2024-12-09 20:41:29,098 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 20:41:29,111 main DEBUG Took 0.010759 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 20:41:29,111 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 20:41:29,112 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 20:41:29,113 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 20:41:29,114 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,123 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 20:41:29,137 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,140 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,141 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,143 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,144 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,145 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,146 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,147 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,148 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,149 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,150 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,151 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:41:29,152 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,152 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 20:41:29,154 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:41:29,156 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 20:41:29,158 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 20:41:29,159 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 20:41:29,160 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 20:41:29,161 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 20:41:29,172 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 20:41:29,175 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 20:41:29,177 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 20:41:29,177 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 20:41:29,177 main DEBUG createAppenders(={Console}) 2024-12-09 20:41:29,178 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-09 20:41:29,179 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 20:41:29,179 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-09 20:41:29,180 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 20:41:29,180 main DEBUG OutputStream closed 2024-12-09 20:41:29,180 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 20:41:29,181 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 20:41:29,181 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-09 20:41:29,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 20:41:29,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 20:41:29,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 20:41:29,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 20:41:29,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 20:41:29,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 20:41:29,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 20:41:29,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 20:41:29,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 20:41:29,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 20:41:29,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 20:41:29,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 20:41:29,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 20:41:29,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 20:41:29,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 20:41:29,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 20:41:29,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 20:41:29,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 20:41:29,255 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 20:41:29,255 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-09 20:41:29,255 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 20:41:29,256 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-09T20:41:29,491 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77 2024-12-09 20:41:29,493 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 20:41:29,494 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T20:41:29,500 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-09T20:41:29,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T20:41:29,531 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe, deleteOnExit=true 2024-12-09T20:41:29,531 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T20:41:29,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/test.cache.data in system properties and HBase conf 2024-12-09T20:41:29,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T20:41:29,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/hadoop.log.dir in system properties and HBase conf 2024-12-09T20:41:29,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T20:41:29,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T20:41:29,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T20:41:29,624 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T20:41:29,706 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T20:41:29,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T20:41:29,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T20:41:29,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T20:41:29,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T20:41:29,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T20:41:29,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T20:41:29,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T20:41:29,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T20:41:29,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T20:41:29,714 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/nfs.dump.dir in system properties and HBase conf 2024-12-09T20:41:29,714 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/java.io.tmpdir in system properties and HBase conf 2024-12-09T20:41:29,714 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T20:41:29,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T20:41:29,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T20:41:30,794 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T20:41:30,896 INFO [Time-limited test {}] log.Log(170): Logging initialized @2460ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T20:41:30,968 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T20:41:31,023 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T20:41:31,043 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T20:41:31,043 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T20:41:31,045 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T20:41:31,057 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T20:41:31,059 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/hadoop.log.dir/,AVAILABLE} 2024-12-09T20:41:31,061 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T20:41:31,263 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/java.io.tmpdir/jetty-localhost-33389-hadoop-hdfs-3_4_1-tests_jar-_-any-7594268005242762154/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T20:41:31,269 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33389} 2024-12-09T20:41:31,270 INFO [Time-limited test {}] server.Server(415): Started @2834ms 2024-12-09T20:41:31,994 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T20:41:32,000 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T20:41:32,001 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T20:41:32,001 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T20:41:32,001 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T20:41:32,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/hadoop.log.dir/,AVAILABLE} 2024-12-09T20:41:32,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T20:41:32,098 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/java.io.tmpdir/jetty-localhost-44319-hadoop-hdfs-3_4_1-tests_jar-_-any-14024469654180940711/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T20:41:32,099 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:44319} 2024-12-09T20:41:32,099 INFO [Time-limited test {}] server.Server(415): Started @3663ms 2024-12-09T20:41:32,147 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T20:41:33,990 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe/data/data2/current/BP-1706245004-172.17.0.2-1733776890246/current, will proceed with Du for space computation calculation, 2024-12-09T20:41:33,990 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe/data/data1/current/BP-1706245004-172.17.0.2-1733776890246/current, will proceed with Du for space computation calculation, 2024-12-09T20:41:34,016 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T20:41:34,066 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b7aa424cb878dfc with lease ID 0x477ce1538854e58b: Processing first storage report for DS-e7cfbe45-3a86-42e1-b03b-5f26496a8087 from datanode DatanodeRegistration(127.0.0.1:38193, datanodeUuid=ffb89470-f156-4695-847b-640b1bfe0f55, infoPort=38787, infoSecurePort=0, ipcPort=39693, storageInfo=lv=-57;cid=testClusterID;nsid=163001773;c=1733776890246) 2024-12-09T20:41:34,068 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b7aa424cb878dfc with lease ID 0x477ce1538854e58b: from storage DS-e7cfbe45-3a86-42e1-b03b-5f26496a8087 node DatanodeRegistration(127.0.0.1:38193, datanodeUuid=ffb89470-f156-4695-847b-640b1bfe0f55, infoPort=38787, infoSecurePort=0, ipcPort=39693, storageInfo=lv=-57;cid=testClusterID;nsid=163001773;c=1733776890246), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-09T20:41:34,069 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b7aa424cb878dfc with lease ID 0x477ce1538854e58b: Processing first storage report for DS-ec917dc6-44fd-4c7f-b88b-75f9971d50cd from datanode DatanodeRegistration(127.0.0.1:38193, datanodeUuid=ffb89470-f156-4695-847b-640b1bfe0f55, infoPort=38787, infoSecurePort=0, ipcPort=39693, storageInfo=lv=-57;cid=testClusterID;nsid=163001773;c=1733776890246) 2024-12-09T20:41:34,069 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b7aa424cb878dfc with lease ID 0x477ce1538854e58b: from storage DS-ec917dc6-44fd-4c7f-b88b-75f9971d50cd node DatanodeRegistration(127.0.0.1:38193, datanodeUuid=ffb89470-f156-4695-847b-640b1bfe0f55, infoPort=38787, infoSecurePort=0, ipcPort=39693, storageInfo=lv=-57;cid=testClusterID;nsid=163001773;c=1733776890246), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T20:41:34,117 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77 2024-12-09T20:41:34,196 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe/zookeeper_0, clientPort=63081, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T20:41:34,207 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63081 2024-12-09T20:41:34,220 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:41:34,223 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:41:34,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741825_1001 (size=7) 2024-12-09T20:41:34,876 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948 with version=8 2024-12-09T20:41:34,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/hbase-staging 2024-12-09T20:41:34,978 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T20:41:35,168 INFO [Time-limited test {}] client.ConnectionUtils(128): master/00c23a7251f8:0 server-side Connection retries=45 2024-12-09T20:41:35,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T20:41:35,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T20:41:35,183 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T20:41:35,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T20:41:35,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T20:41:35,321 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T20:41:35,374 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T20:41:35,383 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T20:41:35,387 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T20:41:35,409 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5073 (auto-detected) 2024-12-09T20:41:35,410 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T20:41:35,428 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42351 2024-12-09T20:41:35,454 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42351 connecting to ZooKeeper ensemble=127.0.0.1:63081 2024-12-09T20:41:35,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423510x0, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T20:41:35,526 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42351-0x1000c4fefa70000 connected 2024-12-09T20:41:35,642 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:41:35,646 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:41:35,655 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T20:41:35,660 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948, hbase.cluster.distributed=false 2024-12-09T20:41:35,681 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T20:41:35,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42351 2024-12-09T20:41:35,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42351 2024-12-09T20:41:35,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42351 2024-12-09T20:41:35,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42351 2024-12-09T20:41:35,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42351 2024-12-09T20:41:35,794 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c23a7251f8:0 server-side Connection retries=45 2024-12-09T20:41:35,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T20:41:35,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T20:41:35,796 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T20:41:35,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T20:41:35,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T20:41:35,799 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T20:41:35,801 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T20:41:35,802 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36519 2024-12-09T20:41:35,804 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36519 connecting to ZooKeeper ensemble=127.0.0.1:63081 2024-12-09T20:41:35,805 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:41:35,808 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:41:35,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365190x0, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T20:41:35,823 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365190x0, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T20:41:35,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36519-0x1000c4fefa70001 connected 2024-12-09T20:41:35,828 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T20:41:35,837 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T20:41:35,839 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T20:41:35,844 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T20:41:35,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36519 2024-12-09T20:41:35,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36519 2024-12-09T20:41:35,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36519 2024-12-09T20:41:35,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36519 2024-12-09T20:41:35,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36519 2024-12-09T20:41:35,858 DEBUG [M:0;00c23a7251f8:42351 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c23a7251f8:42351 2024-12-09T20:41:35,860 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c23a7251f8,42351,1733776895022 2024-12-09T20:41:35,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T20:41:35,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T20:41:35,877 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c23a7251f8,42351,1733776895022 2024-12-09T20:41:35,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:35,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T20:41:35,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:35,907 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T20:41:35,908 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c23a7251f8,42351,1733776895022 from backup master directory 2024-12-09T20:41:35,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c23a7251f8,42351,1733776895022 2024-12-09T20:41:35,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T20:41:35,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T20:41:35,923 WARN [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T20:41:35,923 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c23a7251f8,42351,1733776895022 2024-12-09T20:41:35,926 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T20:41:35,927 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T20:41:35,978 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/hbase.id] with ID: df358591-f077-4d91-8b32-36791192d775 2024-12-09T20:41:35,978 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/.tmp/hbase.id 2024-12-09T20:41:35,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741826_1002 (size=42) 2024-12-09T20:41:35,988 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/.tmp/hbase.id]:[hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/hbase.id] 2024-12-09T20:41:36,033 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:41:36,037 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T20:41:36,053 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-09T20:41:36,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:36,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:36,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741827_1003 (size=196) 2024-12-09T20:41:36,113 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:41:36,116 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T20:41:36,129 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:36,133 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T20:41:36,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741828_1004 (size=1189) 2024-12-09T20:41:36,586 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store 2024-12-09T20:41:36,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741829_1005 (size=34) 2024-12-09T20:41:37,017 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T20:41:37,020 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:41:37,021 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T20:41:37,021 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:41:37,021 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:41:37,023 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T20:41:37,023 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:41:37,023 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:41:37,024 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733776897021Disabling compacts and flushes for region at 1733776897021Disabling writes for close at 1733776897023 (+2 ms)Writing region close event to WAL at 1733776897023Closed at 1733776897023 2024-12-09T20:41:37,026 WARN [master/00c23a7251f8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/.initializing 2024-12-09T20:41:37,026 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/WALs/00c23a7251f8,42351,1733776895022 2024-12-09T20:41:37,033 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T20:41:37,048 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c23a7251f8%2C42351%2C1733776895022, suffix=, logDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/WALs/00c23a7251f8,42351,1733776895022, archiveDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/oldWALs, maxLogs=10 2024-12-09T20:41:37,068 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/WALs/00c23a7251f8,42351,1733776895022/00c23a7251f8%2C42351%2C1733776895022.1733776897052, exclude list is [], retry=0 2024-12-09T20:41:37,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38193,DS-e7cfbe45-3a86-42e1-b03b-5f26496a8087,DISK] 2024-12-09T20:41:37,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T20:41:37,114 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/WALs/00c23a7251f8,42351,1733776895022/00c23a7251f8%2C42351%2C1733776895022.1733776897052 2024-12-09T20:41:37,115 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38787:38787)] 2024-12-09T20:41:37,116 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:41:37,116 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:41:37,120 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,121 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T20:41:37,175 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:37,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:41:37,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T20:41:37,181 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:37,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:41:37,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T20:41:37,186 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:37,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:41:37,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,190 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T20:41:37,190 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:37,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:41:37,192 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,196 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,197 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,203 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,204 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,209 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T20:41:37,213 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:41:37,218 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:41:37,219 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62124578, jitterRate=-0.07427164912223816}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T20:41:37,223 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733776897132Initializing all the Stores at 1733776897134 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776897135 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776897136 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776897136Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776897136Cleaning up temporary data from old regions at 1733776897204 (+68 ms)Region opened successfully at 1733776897223 (+19 ms) 2024-12-09T20:41:37,225 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T20:41:37,252 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d88cf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c23a7251f8/172.17.0.2:0 2024-12-09T20:41:37,278 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T20:41:37,287 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T20:41:37,287 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T20:41:37,289 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T20:41:37,290 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T20:41:37,294 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T20:41:37,295 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T20:41:37,319 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T20:41:37,326 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T20:41:37,401 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T20:41:37,407 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T20:41:37,409 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T20:41:37,422 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T20:41:37,424 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T20:41:37,428 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T20:41:37,437 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T20:41:37,440 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T20:41:37,448 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T20:41:37,470 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T20:41:37,479 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T20:41:37,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T20:41:37,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T20:41:37,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:37,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:37,494 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c23a7251f8,42351,1733776895022, sessionid=0x1000c4fefa70000, setting cluster-up flag (Was=false) 2024-12-09T20:41:37,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:37,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:37,564 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T20:41:37,570 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c23a7251f8,42351,1733776895022 2024-12-09T20:41:37,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:37,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:37,627 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T20:41:37,632 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c23a7251f8,42351,1733776895022 2024-12-09T20:41:37,641 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T20:41:37,651 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(746): ClusterId : df358591-f077-4d91-8b32-36791192d775 2024-12-09T20:41:37,653 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T20:41:37,666 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T20:41:37,666 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T20:41:37,681 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T20:41:37,682 DEBUG [RS:0;00c23a7251f8:36519 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f19d67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c23a7251f8/172.17.0.2:0 2024-12-09T20:41:37,698 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c23a7251f8:36519 2024-12-09T20:41:37,702 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T20:41:37,702 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T20:41:37,702 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T20:41:37,704 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c23a7251f8,42351,1733776895022 with port=36519, startcode=1733776895755 2024-12-09T20:41:37,705 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T20:41:37,712 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T20:41:37,715 DEBUG [RS:0;00c23a7251f8:36519 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T20:41:37,718 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T20:41:37,722 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c23a7251f8,42351,1733776895022 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T20:41:37,729 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c23a7251f8:0, corePoolSize=5, maxPoolSize=5 2024-12-09T20:41:37,729 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c23a7251f8:0, corePoolSize=5, maxPoolSize=5 2024-12-09T20:41:37,729 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c23a7251f8:0, corePoolSize=5, maxPoolSize=5 2024-12-09T20:41:37,729 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c23a7251f8:0, corePoolSize=5, maxPoolSize=5 2024-12-09T20:41:37,729 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c23a7251f8:0, corePoolSize=10, maxPoolSize=10 2024-12-09T20:41:37,729 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,730 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c23a7251f8:0, corePoolSize=2, maxPoolSize=2 2024-12-09T20:41:37,730 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,736 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733776927736 2024-12-09T20:41:37,738 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T20:41:37,738 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T20:41:37,738 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T20:41:37,739 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T20:41:37,742 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T20:41:37,743 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T20:41:37,743 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T20:41:37,743 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T20:41:37,744 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:37,745 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T20:41:37,744 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,747 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T20:41:37,749 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T20:41:37,750 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41517, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T20:41:37,750 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T20:41:37,752 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T20:41:37,752 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T20:41:37,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741831_1007 (size=1321) 2024-12-09T20:41:37,755 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c23a7251f8:0:becomeActiveMaster-HFileCleaner.large.0-1733776897754,5,FailOnTimeoutGroup] 2024-12-09T20:41:37,755 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c23a7251f8:0:becomeActiveMaster-HFileCleaner.small.0-1733776897755,5,FailOnTimeoutGroup] 2024-12-09T20:41:37,755 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,756 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T20:41:37,757 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,757 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:37,757 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,777 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T20:41:37,777 WARN [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T20:41:37,879 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c23a7251f8,42351,1733776895022 with port=36519, startcode=1733776895755 2024-12-09T20:41:37,883 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c23a7251f8,36519,1733776895755 2024-12-09T20:41:37,887 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42351 {}] master.ServerManager(517): Registering regionserver=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:37,896 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948 2024-12-09T20:41:37,896 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42427 2024-12-09T20:41:37,896 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T20:41:37,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T20:41:37,912 DEBUG [RS:0;00c23a7251f8:36519 {}] zookeeper.ZKUtil(111): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c23a7251f8,36519,1733776895755 2024-12-09T20:41:37,912 WARN [RS:0;00c23a7251f8:36519 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T20:41:37,912 INFO [RS:0;00c23a7251f8:36519 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T20:41:37,912 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/WALs/00c23a7251f8,36519,1733776895755 2024-12-09T20:41:37,915 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c23a7251f8,36519,1733776895755] 2024-12-09T20:41:37,937 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T20:41:37,948 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T20:41:37,952 INFO [RS:0;00c23a7251f8:36519 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T20:41:37,952 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,953 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T20:41:37,958 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T20:41:37,959 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,959 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,959 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,959 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,959 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,960 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,960 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c23a7251f8:0, corePoolSize=2, maxPoolSize=2 2024-12-09T20:41:37,960 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,960 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,960 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,960 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,960 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,961 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c23a7251f8:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:41:37,961 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c23a7251f8:0, corePoolSize=3, maxPoolSize=3 2024-12-09T20:41:37,961 DEBUG [RS:0;00c23a7251f8:36519 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0, corePoolSize=3, maxPoolSize=3 2024-12-09T20:41:37,962 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,962 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,962 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,962 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,962 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,962 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c23a7251f8,36519,1733776895755-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T20:41:37,979 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T20:41:37,981 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c23a7251f8,36519,1733776895755-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,981 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,981 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.Replication(171): 00c23a7251f8,36519,1733776895755 started 2024-12-09T20:41:37,996 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:37,996 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1482): Serving as 00c23a7251f8,36519,1733776895755, RpcServer on 00c23a7251f8/172.17.0.2:36519, sessionid=0x1000c4fefa70001 2024-12-09T20:41:37,997 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T20:41:37,997 DEBUG [RS:0;00c23a7251f8:36519 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c23a7251f8,36519,1733776895755 2024-12-09T20:41:37,998 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c23a7251f8,36519,1733776895755' 2024-12-09T20:41:37,998 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T20:41:37,999 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T20:41:38,000 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T20:41:38,000 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T20:41:38,000 DEBUG [RS:0;00c23a7251f8:36519 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c23a7251f8,36519,1733776895755 2024-12-09T20:41:38,000 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c23a7251f8,36519,1733776895755' 2024-12-09T20:41:38,000 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T20:41:38,001 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T20:41:38,001 DEBUG [RS:0;00c23a7251f8:36519 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T20:41:38,001 INFO [RS:0;00c23a7251f8:36519 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T20:41:38,001 INFO [RS:0;00c23a7251f8:36519 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T20:41:38,107 INFO [RS:0;00c23a7251f8:36519 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T20:41:38,110 INFO [RS:0;00c23a7251f8:36519 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c23a7251f8%2C36519%2C1733776895755, suffix=, logDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/WALs/00c23a7251f8,36519,1733776895755, archiveDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/oldWALs, maxLogs=32 2024-12-09T20:41:38,127 DEBUG [RS:0;00c23a7251f8:36519 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/WALs/00c23a7251f8,36519,1733776895755/00c23a7251f8%2C36519%2C1733776895755.1733776898113, exclude list is [], retry=0 2024-12-09T20:41:38,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38193,DS-e7cfbe45-3a86-42e1-b03b-5f26496a8087,DISK] 2024-12-09T20:41:38,134 INFO [RS:0;00c23a7251f8:36519 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/WALs/00c23a7251f8,36519,1733776895755/00c23a7251f8%2C36519%2C1733776895755.1733776898113 2024-12-09T20:41:38,134 DEBUG [RS:0;00c23a7251f8:36519 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38787:38787)] 2024-12-09T20:41:38,157 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T20:41:38,158 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948 2024-12-09T20:41:38,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741833_1009 (size=32) 2024-12-09T20:41:38,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:41:38,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T20:41:38,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T20:41:38,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:38,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:41:38,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T20:41:38,586 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T20:41:38,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:38,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:41:38,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T20:41:38,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T20:41:38,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:38,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:41:38,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T20:41:38,593 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T20:41:38,593 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:38,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:41:38,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T20:41:38,595 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740 2024-12-09T20:41:38,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740 2024-12-09T20:41:38,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T20:41:38,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T20:41:38,600 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:41:38,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T20:41:38,606 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:41:38,607 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72385842, jitterRate=0.07863309979438782}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:41:38,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733776898574Initializing all the Stores at 1733776898578 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776898578Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776898579 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776898579Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776898579Cleaning up temporary data from old regions at 1733776898599 (+20 ms)Region opened successfully at 1733776898609 (+10 ms) 2024-12-09T20:41:38,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T20:41:38,609 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T20:41:38,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T20:41:38,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T20:41:38,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T20:41:38,610 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T20:41:38,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733776898609Disabling compacts and flushes for region at 1733776898609Disabling writes for close at 1733776898609Writing region close event to WAL at 1733776898610 (+1 ms)Closed at 1733776898610 2024-12-09T20:41:38,613 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T20:41:38,614 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T20:41:38,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T20:41:38,626 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T20:41:38,629 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T20:41:38,786 DEBUG [00c23a7251f8:42351 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T20:41:38,800 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:38,806 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c23a7251f8,36519,1733776895755, state=OPENING 2024-12-09T20:41:38,853 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T20:41:38,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:38,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:41:38,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T20:41:38,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T20:41:38,869 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T20:41:38,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:41:39,051 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T20:41:39,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53715, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T20:41:39,064 INFO [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T20:41:39,064 INFO [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T20:41:39,065 INFO [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T20:41:39,068 INFO [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c23a7251f8%2C36519%2C1733776895755.meta, suffix=.meta, logDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/WALs/00c23a7251f8,36519,1733776895755, archiveDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/oldWALs, maxLogs=32 2024-12-09T20:41:39,081 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/WALs/00c23a7251f8,36519,1733776895755/00c23a7251f8%2C36519%2C1733776895755.meta.1733776899069.meta, exclude list is [], retry=0 2024-12-09T20:41:39,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38193,DS-e7cfbe45-3a86-42e1-b03b-5f26496a8087,DISK] 2024-12-09T20:41:39,088 INFO [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/WALs/00c23a7251f8,36519,1733776895755/00c23a7251f8%2C36519%2C1733776895755.meta.1733776899069.meta 2024-12-09T20:41:39,089 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38787:38787)] 2024-12-09T20:41:39,089 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:41:39,091 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T20:41:39,093 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T20:41:39,097 INFO [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T20:41:39,101 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T20:41:39,102 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:41:39,102 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T20:41:39,102 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T20:41:39,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T20:41:39,107 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T20:41:39,107 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:39,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:41:39,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T20:41:39,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T20:41:39,110 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:39,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:41:39,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T20:41:39,113 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T20:41:39,113 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:39,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:41:39,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T20:41:39,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T20:41:39,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:39,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:41:39,117 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T20:41:39,119 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740 2024-12-09T20:41:39,121 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740 2024-12-09T20:41:39,124 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T20:41:39,124 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T20:41:39,125 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:41:39,127 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T20:41:39,129 INFO [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74979523, jitterRate=0.1172819584608078}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:41:39,129 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T20:41:39,130 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733776899103Writing region info on filesystem at 1733776899103Initializing all the Stores at 1733776899105 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776899105Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776899105Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776899105Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776899105Cleaning up temporary data from old regions at 1733776899124 (+19 ms)Running coprocessor post-open hooks at 1733776899129 (+5 ms)Region opened successfully at 1733776899130 (+1 ms) 2024-12-09T20:41:39,136 INFO [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733776899044 2024-12-09T20:41:39,146 DEBUG [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T20:41:39,147 INFO [RS_OPEN_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T20:41:39,148 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:39,151 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c23a7251f8,36519,1733776895755, state=OPEN 2024-12-09T20:41:39,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T20:41:39,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T20:41:39,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T20:41:39,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T20:41:39,192 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:39,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T20:41:39,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c23a7251f8,36519,1733776895755 in 321 msec 2024-12-09T20:41:39,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T20:41:39,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 580 msec 2024-12-09T20:41:39,207 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T20:41:39,207 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T20:41:39,226 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:39,227 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:39,246 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:39,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47203, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:39,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5980 sec 2024-12-09T20:41:39,269 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733776899269, completionTime=-1 2024-12-09T20:41:39,272 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T20:41:39,272 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T20:41:39,295 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T20:41:39,295 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733776959295 2024-12-09T20:41:39,295 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733777019295 2024-12-09T20:41:39,295 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-09T20:41:39,297 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c23a7251f8,42351,1733776895022-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:39,297 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c23a7251f8,42351,1733776895022-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:39,297 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c23a7251f8,42351,1733776895022-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:39,299 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c23a7251f8:42351, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:39,299 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:39,300 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:39,306 DEBUG [master/00c23a7251f8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T20:41:39,343 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.420sec 2024-12-09T20:41:39,344 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T20:41:39,346 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T20:41:39,347 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T20:41:39,348 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T20:41:39,348 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T20:41:39,349 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c23a7251f8,42351,1733776895022-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T20:41:39,349 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c23a7251f8,42351,1733776895022-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T20:41:39,359 DEBUG [master/00c23a7251f8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T20:41:39,359 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T20:41:39,363 INFO [master/00c23a7251f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c23a7251f8,42351,1733776895022-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:41:39,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4346ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:39,370 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T20:41:39,370 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T20:41:39,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:39,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:39,386 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37bc83f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:39,395 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:39,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbb1cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:39,398 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:39,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:39,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c23a7251f8,42351,1733776895022 2024-12-09T20:41:39,436 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=3717 2024-12-09T20:41:39,444 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T20:41:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c23a7251f8,42351,1733776895022 2024-12-09T20:41:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ce75c02 2024-12-09T20:41:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T20:41:39,454 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T20:41:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T20:41:39,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:41:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T20:41:39,469 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:41:39,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-09T20:41:39,472 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:39,475 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:41:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T20:41:39,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741835_1011 (size=1061) 2024-12-09T20:41:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T20:41:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T20:41:39,889 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948 2024-12-09T20:41:39,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741836_1012 (size=53) 2024-12-09T20:41:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T20:41:40,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:41:40,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dd7288626d8fbb1461f9ccf6551f1bf6, disabling compactions & flushes 2024-12-09T20:41:40,301 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:40,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:40,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. after waiting 0 ms 2024-12-09T20:41:40,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:40,301 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:40,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dd7288626d8fbb1461f9ccf6551f1bf6: Waiting for close lock at 1733776900300Disabling compacts and flushes for region at 1733776900300Disabling writes for close at 1733776900301 (+1 ms)Writing region close event to WAL at 1733776900301Closed at 1733776900301 2024-12-09T20:41:40,303 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:41:40,308 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733776900303"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776900303"}]},"ts":"1733776900303"} 2024-12-09T20:41:40,313 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:41:40,315 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:41:40,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776900315"}]},"ts":"1733776900315"} 2024-12-09T20:41:40,322 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T20:41:40,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd7288626d8fbb1461f9ccf6551f1bf6, ASSIGN}] 2024-12-09T20:41:40,326 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd7288626d8fbb1461f9ccf6551f1bf6, ASSIGN 2024-12-09T20:41:40,328 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd7288626d8fbb1461f9ccf6551f1bf6, ASSIGN; state=OFFLINE, location=00c23a7251f8,36519,1733776895755; forceNewPlan=false, retain=false 2024-12-09T20:41:40,481 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dd7288626d8fbb1461f9ccf6551f1bf6, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:40,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd7288626d8fbb1461f9ccf6551f1bf6, ASSIGN because future has completed 2024-12-09T20:41:40,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:41:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T20:41:40,655 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:40,656 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:41:40,656 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,656 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:41:40,657 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,657 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,660 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,663 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:41:40,663 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd7288626d8fbb1461f9ccf6551f1bf6 columnFamilyName A 2024-12-09T20:41:40,664 DEBUG [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:40,665 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] regionserver.HStore(327): Store=dd7288626d8fbb1461f9ccf6551f1bf6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:41:40,665 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,667 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:41:40,667 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd7288626d8fbb1461f9ccf6551f1bf6 columnFamilyName B 2024-12-09T20:41:40,667 DEBUG [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:40,668 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] regionserver.HStore(327): Store=dd7288626d8fbb1461f9ccf6551f1bf6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:41:40,668 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,670 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:41:40,670 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd7288626d8fbb1461f9ccf6551f1bf6 columnFamilyName C 2024-12-09T20:41:40,671 DEBUG [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:41:40,671 INFO [StoreOpener-dd7288626d8fbb1461f9ccf6551f1bf6-1 {}] regionserver.HStore(327): Store=dd7288626d8fbb1461f9ccf6551f1bf6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:41:40,672 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:40,672 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,673 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,674 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,676 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,676 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,677 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:41:40,679 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,683 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:41:40,684 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dd7288626d8fbb1461f9ccf6551f1bf6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71852918, jitterRate=0.07069191336631775}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:41:40,684 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:40,685 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dd7288626d8fbb1461f9ccf6551f1bf6: Running coprocessor pre-open hook at 1733776900657Writing region info on filesystem at 1733776900657Initializing all the Stores at 1733776900659 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776900659Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776900659Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776900659Cleaning up temporary data from old regions at 1733776900676 (+17 ms)Running coprocessor post-open hooks at 1733776900684 (+8 ms)Region opened successfully at 1733776900685 (+1 ms) 2024-12-09T20:41:40,687 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., pid=6, masterSystemTime=1733776900646 2024-12-09T20:41:40,690 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:40,690 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:40,691 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dd7288626d8fbb1461f9ccf6551f1bf6, regionState=OPEN, openSeqNum=2, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:40,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:41:40,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T20:41:40,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 in 206 msec 2024-12-09T20:41:40,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T20:41:40,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd7288626d8fbb1461f9ccf6551f1bf6, ASSIGN in 377 msec 2024-12-09T20:41:40,706 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:41:40,707 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776900707"}]},"ts":"1733776900707"} 2024-12-09T20:41:40,710 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T20:41:40,711 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:41:40,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2500 sec 2024-12-09T20:41:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T20:41:41,625 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:41,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775e68a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,636 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,639 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@640890e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,641 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,647 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T20:41:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c23a7251f8,42351,1733776895022 2024-12-09T20:41:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4df731e6 2024-12-09T20:41:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T20:41:41,654 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T20:41:41,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,660 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1613decc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,663 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46fadb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,667 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ac9955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,670 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230faddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,673 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ccba464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2523f4b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,680 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3401ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,683 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3e565e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,686 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424b4efb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,689 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cbec0b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,693 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c07a1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,695 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39170b45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,698 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c39882a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40cd31f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,704 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df7e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,707 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b6e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:41:41,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:41:41,710 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:41:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:41:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:41:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1058791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:41:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:41:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:41:41,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:41:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13268882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:41:41,717 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,717 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,717 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,718 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,718 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,719 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,719 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,721 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-09T20:41:41,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,730 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:41:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:41:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:41:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:41:41,734 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:41,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:41,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:41:41,738 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:41:41,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:41:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:41:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:41:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T20:41:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:41:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:41:41,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:41:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:41:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:41,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:41:41,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:41,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:41,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:41,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:41,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:41,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T20:41:41,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776961893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:41,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:41,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/272d1ac76686429fb70d01f03ab734da is 50, key is test_row_0/A:col10/1733776901792/Put/seqid=0 2024-12-09T20:41:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776961900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:41,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T20:41:41,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776961905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:41,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776961908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776961913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:41,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:41,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:41,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741837_1013 (size=12001) 2024-12-09T20:41:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:41,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/272d1ac76686429fb70d01f03ab734da 2024-12-09T20:41:42,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776962032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776962032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776962043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776962043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776962043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/1300c7ec0c9248a59c642078fb25f5e2 is 50, key is test_row_0/B:col10/1733776901792/Put/seqid=0 2024-12-09T20:41:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T20:41:42,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741838_1014 (size=12001) 2024-12-09T20:41:42,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/1300c7ec0c9248a59c642078fb25f5e2 2024-12-09T20:41:42,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T20:41:42,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:42,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:42,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:42,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:42,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:42,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/73236cbd0de6440081cc78a185862842 is 50, key is test_row_0/C:col10/1733776901792/Put/seqid=0 2024-12-09T20:41:42,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741839_1015 (size=12001) 2024-12-09T20:41:42,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/73236cbd0de6440081cc78a185862842 2024-12-09T20:41:42,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/272d1ac76686429fb70d01f03ab734da as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/272d1ac76686429fb70d01f03ab734da 2024-12-09T20:41:42,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/272d1ac76686429fb70d01f03ab734da, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T20:41:42,219 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-09T20:41:42,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/1300c7ec0c9248a59c642078fb25f5e2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/1300c7ec0c9248a59c642078fb25f5e2 2024-12-09T20:41:42,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/1300c7ec0c9248a59c642078fb25f5e2, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T20:41:42,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/73236cbd0de6440081cc78a185862842 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73236cbd0de6440081cc78a185862842 2024-12-09T20:41:42,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776962243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776962243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776962252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73236cbd0de6440081cc78a185862842, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T20:41:42,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for dd7288626d8fbb1461f9ccf6551f1bf6 in 469ms, sequenceid=13, compaction requested=false 2024-12-09T20:41:42,269 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T20:41:42,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:42,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T20:41:42,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776901882, max timestamp=1733776902271Num uniques 859; 2024-12-09T20:41:42,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776901882, max timestamp=1733776902271Num uniques 859; 2024-12-09T20:41:42,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:42,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:41:42,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:41:42,289 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-09T20:41:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:42,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:42,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:42,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:42,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:42,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/a369d3da936a40088f682eecbbea2b3f is 50, key is test_row_0/A:col10/1733776902271/Put/seqid=0 2024-12-09T20:41:42,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741840_1016 (size=12001) 2024-12-09T20:41:42,346 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/a369d3da936a40088f682eecbbea2b3f 2024-12-09T20:41:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T20:41:42,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0b3ac6a3e956441aa3d5936f480f7d6a is 50, key is test_row_0/B:col10/1733776902271/Put/seqid=0 2024-12-09T20:41:42,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741841_1017 (size=12001) 2024-12-09T20:41:42,407 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0b3ac6a3e956441aa3d5936f480f7d6a 2024-12-09T20:41:42,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2a09840e449b42ec8ffb7a490ce6b794 is 50, key is test_row_0/C:col10/1733776902271/Put/seqid=0 2024-12-09T20:41:42,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741842_1018 (size=12001) 2024-12-09T20:41:42,443 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2a09840e449b42ec8ffb7a490ce6b794 2024-12-09T20:41:42,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/a369d3da936a40088f682eecbbea2b3f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a369d3da936a40088f682eecbbea2b3f 2024-12-09T20:41:42,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776902384, max timestamp=1733776902461Num uniques 859; 2024-12-09T20:41:42,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:41:42,483 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a369d3da936a40088f682eecbbea2b3f, entries=150, sequenceid=35, filesize=11.7 K 2024-12-09T20:41:42,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776902384, max timestamp=1733776902461Num uniques 859; 2024-12-09T20:41:42,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:41:42,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0b3ac6a3e956441aa3d5936f480f7d6a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0b3ac6a3e956441aa3d5936f480f7d6a 2024-12-09T20:41:42,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776902384, max timestamp=1733776902461Num uniques 859; 2024-12-09T20:41:42,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:41:42,508 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0b3ac6a3e956441aa3d5936f480f7d6a, entries=150, sequenceid=35, filesize=11.7 K 2024-12-09T20:41:42,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2a09840e449b42ec8ffb7a490ce6b794 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2a09840e449b42ec8ffb7a490ce6b794 2024-12-09T20:41:42,536 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2a09840e449b42ec8ffb7a490ce6b794, entries=150, sequenceid=35, filesize=11.7 K 2024-12-09T20:41:42,541 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=67.09 KB/68700 for dd7288626d8fbb1461f9ccf6551f1bf6 in 252ms, sequenceid=35, compaction requested=false 2024-12-09T20:41:42,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:42,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:42,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-09T20:41:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-09T20:41:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:42,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=73.80 KB heapSize=205.69 KB 2024-12-09T20:41:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:42,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T20:41:42,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 809 msec 2024-12-09T20:41:42,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 837 msec 2024-12-09T20:41:42,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/fcaa307c704d487e8541fa90270c6655 is 50, key is test_row_0/A:col10/1733776902520/Put/seqid=0 2024-12-09T20:41:42,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741843_1019 (size=16681) 2024-12-09T20:41:42,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/fcaa307c704d487e8541fa90270c6655 2024-12-09T20:41:42,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/349522686b0845adb4b906faba2c957e is 50, key is test_row_0/B:col10/1733776902520/Put/seqid=0 2024-12-09T20:41:42,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776962615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776962635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741844_1020 (size=12001) 2024-12-09T20:41:42,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776962636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776962641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/349522686b0845adb4b906faba2c957e 2024-12-09T20:41:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776962641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2378d1c58b0441b88922afae9ecf16dc is 50, key is test_row_0/C:col10/1733776902520/Put/seqid=0 2024-12-09T20:41:42,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741845_1021 (size=12001) 2024-12-09T20:41:42,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2378d1c58b0441b88922afae9ecf16dc 2024-12-09T20:41:42,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/fcaa307c704d487e8541fa90270c6655 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fcaa307c704d487e8541fa90270c6655 2024-12-09T20:41:42,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fcaa307c704d487e8541fa90270c6655, entries=250, sequenceid=63, filesize=16.3 K 2024-12-09T20:41:42,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/349522686b0845adb4b906faba2c957e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/349522686b0845adb4b906faba2c957e 2024-12-09T20:41:42,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/349522686b0845adb4b906faba2c957e, entries=150, sequenceid=63, filesize=11.7 K 2024-12-09T20:41:42,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2378d1c58b0441b88922afae9ecf16dc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2378d1c58b0441b88922afae9ecf16dc 2024-12-09T20:41:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776962752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776962762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2378d1c58b0441b88922afae9ecf16dc, entries=150, sequenceid=63, filesize=11.7 K 2024-12-09T20:41:42,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776962764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776962763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=114.05 KB/116790 for dd7288626d8fbb1461f9ccf6551f1bf6 in 223ms, sequenceid=63, compaction requested=true 2024-12-09T20:41:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:42,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:42,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:42,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:42,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:42,779 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:42,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:42,779 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:42,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T20:41:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:42,786 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:42,786 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:42,788 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:41:42,788 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:41:42,788 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:42,788 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:42,789 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/1300c7ec0c9248a59c642078fb25f5e2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0b3ac6a3e956441aa3d5936f480f7d6a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/349522686b0845adb4b906faba2c957e] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=35.2 K 2024-12-09T20:41:42,789 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/272d1ac76686429fb70d01f03ab734da, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a369d3da936a40088f682eecbbea2b3f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fcaa307c704d487e8541fa90270c6655] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=39.7 K 2024-12-09T20:41:42,791 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 1300c7ec0c9248a59c642078fb25f5e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733776901785 2024-12-09T20:41:42,791 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272d1ac76686429fb70d01f03ab734da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733776901785 2024-12-09T20:41:42,792 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b3ac6a3e956441aa3d5936f480f7d6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733776901882 2024-12-09T20:41:42,792 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting a369d3da936a40088f682eecbbea2b3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733776901882 2024-12-09T20:41:42,793 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 349522686b0845adb4b906faba2c957e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733776902520 2024-12-09T20:41:42,794 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcaa307c704d487e8541fa90270c6655, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733776902501 2024-12-09T20:41:42,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/50456e1520654fcb9706eb3dbb733868 is 50, key is test_row_0/A:col10/1733776902773/Put/seqid=0 2024-12-09T20:41:42,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741846_1022 (size=14341) 2024-12-09T20:41:42,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/50456e1520654fcb9706eb3dbb733868 2024-12-09T20:41:42,833 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:42,835 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/ff0cae8179584951b35e5eb73bf86354 is 50, key is test_row_0/A:col10/1733776902520/Put/seqid=0 2024-12-09T20:41:42,837 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:42,838 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/6c734905a4904bbaa7def8933defff93 is 50, key is test_row_0/B:col10/1733776902520/Put/seqid=0 2024-12-09T20:41:42,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/00b795e7165848a9bdc18292f6263009 is 50, key is test_row_0/B:col10/1733776902773/Put/seqid=0 2024-12-09T20:41:42,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741847_1023 (size=12104) 2024-12-09T20:41:42,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741848_1024 (size=12104) 2024-12-09T20:41:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T20:41:42,883 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:42,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:42,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741849_1025 (size=12001) 2024-12-09T20:41:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-09T20:41:42,886 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/ff0cae8179584951b35e5eb73bf86354 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/ff0cae8179584951b35e5eb73bf86354 2024-12-09T20:41:42,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/00b795e7165848a9bdc18292f6263009 2024-12-09T20:41:42,888 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:42,891 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:42,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T20:41:42,903 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/6c734905a4904bbaa7def8933defff93 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6c734905a4904bbaa7def8933defff93 2024-12-09T20:41:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776962894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,924 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into 6c734905a4904bbaa7def8933defff93(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:42,924 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into ff0cae8179584951b35e5eb73bf86354(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:42,924 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:42,924 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:42,927 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=13, startTime=1733776902779; duration=0sec 2024-12-09T20:41:42,927 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=13, startTime=1733776902775; duration=0sec 2024-12-09T20:41:42,927 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:42,927 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:42,928 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:42,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/3d12d889cff240fcbd8b46d20fa5b61b is 50, key is test_row_0/C:col10/1733776902773/Put/seqid=0 2024-12-09T20:41:42,929 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:42,929 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:42,930 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:42,930 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:41:42,930 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:42,931 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73236cbd0de6440081cc78a185862842, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2a09840e449b42ec8ffb7a490ce6b794, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2378d1c58b0441b88922afae9ecf16dc] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=35.2 K 2024-12-09T20:41:42,932 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 73236cbd0de6440081cc78a185862842, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733776901785 2024-12-09T20:41:42,933 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a09840e449b42ec8ffb7a490ce6b794, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733776901882 2024-12-09T20:41:42,934 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2378d1c58b0441b88922afae9ecf16dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733776902520 2024-12-09T20:41:42,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741850_1026 (size=12001) 2024-12-09T20:41:42,968 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#14 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:42,969 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/40010b2b3e8142dba3ce60b44e29ae0c is 50, key is test_row_0/C:col10/1733776902520/Put/seqid=0 2024-12-09T20:41:42,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776962973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776962973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776962982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:42,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776962982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T20:41:43,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741851_1027 (size=12104) 2024-12-09T20:41:43,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776963011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,025 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/40010b2b3e8142dba3ce60b44e29ae0c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/40010b2b3e8142dba3ce60b44e29ae0c 2024-12-09T20:41:43,043 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into 40010b2b3e8142dba3ce60b44e29ae0c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:43,043 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:43,044 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=13, startTime=1733776902779; duration=0sec 2024-12-09T20:41:43,044 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:43,044 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:43,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T20:41:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:43,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:43,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T20:41:43,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:43,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:43,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:43,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:43,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T20:41:43,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776963222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776963292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776963292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776963292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776963293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/3d12d889cff240fcbd8b46d20fa5b61b 2024-12-09T20:41:43,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T20:41:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:43,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:43,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/50456e1520654fcb9706eb3dbb733868 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/50456e1520654fcb9706eb3dbb733868 2024-12-09T20:41:43,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/50456e1520654fcb9706eb3dbb733868, entries=200, sequenceid=84, filesize=14.0 K 2024-12-09T20:41:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/00b795e7165848a9bdc18292f6263009 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/00b795e7165848a9bdc18292f6263009 2024-12-09T20:41:43,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/00b795e7165848a9bdc18292f6263009, entries=150, sequenceid=84, filesize=11.7 K 2024-12-09T20:41:43,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/3d12d889cff240fcbd8b46d20fa5b61b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3d12d889cff240fcbd8b46d20fa5b61b 2024-12-09T20:41:43,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3d12d889cff240fcbd8b46d20fa5b61b, entries=150, sequenceid=84, filesize=11.7 K 2024-12-09T20:41:43,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for dd7288626d8fbb1461f9ccf6551f1bf6 in 655ms, sequenceid=84, compaction requested=false 2024-12-09T20:41:43,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:43,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T20:41:43,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:43,520 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T20:41:43,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:43,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:43,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:43,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:43,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:43,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T20:41:43,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/effb4933321c4108a14fb40fba074195 is 50, key is test_row_0/A:col10/1733776902824/Put/seqid=0 2024-12-09T20:41:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:43,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741852_1028 (size=12001) 2024-12-09T20:41:43,581 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/effb4933321c4108a14fb40fba074195 2024-12-09T20:41:43,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/f7162a5ddad9457b9ca030c4de6eb8b3 is 50, key is test_row_0/B:col10/1733776902824/Put/seqid=0 2024-12-09T20:41:43,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741853_1029 (size=12001) 2024-12-09T20:41:43,643 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/f7162a5ddad9457b9ca030c4de6eb8b3 2024-12-09T20:41:43,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733776903631, max timestamp=1733776903662Num uniques 859; 2024-12-09T20:41:43,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733776903631, max timestamp=1733776903662Num uniques 859; 2024-12-09T20:41:43,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:41:43,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:41:43,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733776903631, max timestamp=1733776903662Num uniques 859; 2024-12-09T20:41:43,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:41:43,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6e94d56fea134f23adda3bd0536a5348 is 50, key is test_row_0/C:col10/1733776902824/Put/seqid=0 2024-12-09T20:41:43,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741854_1030 (size=12001) 2024-12-09T20:41:43,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776963781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776963801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776963802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776963802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776963803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776963892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:43,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:43,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:43,932 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T20:41:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T20:41:44,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776964101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:44,125 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6e94d56fea134f23adda3bd0536a5348 2024-12-09T20:41:44,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/effb4933321c4108a14fb40fba074195 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/effb4933321c4108a14fb40fba074195 2024-12-09T20:41:44,148 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/effb4933321c4108a14fb40fba074195, entries=150, sequenceid=98, filesize=11.7 K 2024-12-09T20:41:44,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/f7162a5ddad9457b9ca030c4de6eb8b3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f7162a5ddad9457b9ca030c4de6eb8b3 2024-12-09T20:41:44,167 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f7162a5ddad9457b9ca030c4de6eb8b3, entries=150, sequenceid=98, filesize=11.7 K 2024-12-09T20:41:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6e94d56fea134f23adda3bd0536a5348 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6e94d56fea134f23adda3bd0536a5348 2024-12-09T20:41:44,185 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6e94d56fea134f23adda3bd0536a5348, entries=150, sequenceid=98, filesize=11.7 K 2024-12-09T20:41:44,188 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dd7288626d8fbb1461f9ccf6551f1bf6 in 668ms, sequenceid=98, compaction requested=true 2024-12-09T20:41:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-09T20:41:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-09T20:41:44,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-09T20:41:44,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3000 sec 2024-12-09T20:41:44,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3580 sec 2024-12-09T20:41:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:44,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T20:41:44,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:44,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:44,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:44,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:44,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:44,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:44,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/d68f1e8df5e5443f9bd05c12d292a226 is 50, key is test_row_0/A:col10/1733776903765/Put/seqid=0 2024-12-09T20:41:44,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741855_1031 (size=12101) 2024-12-09T20:41:44,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/d68f1e8df5e5443f9bd05c12d292a226 2024-12-09T20:41:44,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776964465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:44,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0c1c47db05964bd9b7b2aaf114f24d6f is 50, key is test_row_0/B:col10/1733776903765/Put/seqid=0 2024-12-09T20:41:44,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741856_1032 (size=12101) 2024-12-09T20:41:44,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0c1c47db05964bd9b7b2aaf114f24d6f 2024-12-09T20:41:44,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/17171fa2fc6940b9a071547cebf1ef8a is 50, key is test_row_0/C:col10/1733776903765/Put/seqid=0 2024-12-09T20:41:44,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741857_1033 (size=12101) 2024-12-09T20:41:44,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/17171fa2fc6940b9a071547cebf1ef8a 2024-12-09T20:41:44,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/d68f1e8df5e5443f9bd05c12d292a226 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d68f1e8df5e5443f9bd05c12d292a226 2024-12-09T20:41:44,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d68f1e8df5e5443f9bd05c12d292a226, entries=150, sequenceid=134, filesize=11.8 K 2024-12-09T20:41:44,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0c1c47db05964bd9b7b2aaf114f24d6f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0c1c47db05964bd9b7b2aaf114f24d6f 2024-12-09T20:41:44,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776964581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:44,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0c1c47db05964bd9b7b2aaf114f24d6f, entries=150, sequenceid=134, filesize=11.8 K 2024-12-09T20:41:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:44,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/17171fa2fc6940b9a071547cebf1ef8a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/17171fa2fc6940b9a071547cebf1ef8a 2024-12-09T20:41:44,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/17171fa2fc6940b9a071547cebf1ef8a, entries=150, sequenceid=134, filesize=11.8 K 2024-12-09T20:41:44,614 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:41:44,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for dd7288626d8fbb1461f9ccf6551f1bf6 in 197ms, sequenceid=134, compaction requested=true 2024-12-09T20:41:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:44,615 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:41:44,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:44,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:44,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:44,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:44,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:41:44,617 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:41:44,620 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50547 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:41:44,620 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:41:44,620 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:44,620 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:41:44,621 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:41:44,621 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:44,621 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/ff0cae8179584951b35e5eb73bf86354, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/50456e1520654fcb9706eb3dbb733868, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/effb4933321c4108a14fb40fba074195, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d68f1e8df5e5443f9bd05c12d292a226] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=49.4 K 2024-12-09T20:41:44,621 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6c734905a4904bbaa7def8933defff93, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/00b795e7165848a9bdc18292f6263009, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f7162a5ddad9457b9ca030c4de6eb8b3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0c1c47db05964bd9b7b2aaf114f24d6f] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=47.1 K 2024-12-09T20:41:44,624 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff0cae8179584951b35e5eb73bf86354, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733776902520 2024-12-09T20:41:44,624 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c734905a4904bbaa7def8933defff93, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733776902520 2024-12-09T20:41:44,625 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b795e7165848a9bdc18292f6263009, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733776902610 2024-12-09T20:41:44,625 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50456e1520654fcb9706eb3dbb733868, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733776902610 2024-12-09T20:41:44,629 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting effb4933321c4108a14fb40fba074195, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733776902824 2024-12-09T20:41:44,629 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f7162a5ddad9457b9ca030c4de6eb8b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733776902824 2024-12-09T20:41:44,630 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d68f1e8df5e5443f9bd05c12d292a226, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733776903745 2024-12-09T20:41:44,632 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c1c47db05964bd9b7b2aaf114f24d6f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733776903745 2024-12-09T20:41:44,665 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:44,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/095424b9641342c0b43795b15ebdc0ed is 50, key is test_row_0/A:col10/1733776903765/Put/seqid=0 2024-12-09T20:41:44,707 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#22 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:44,708 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/56c3e1c7aaa342e382cdc6bf9fae2422 is 50, key is test_row_0/B:col10/1733776903765/Put/seqid=0 2024-12-09T20:41:44,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741858_1034 (size=12341) 2024-12-09T20:41:44,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741859_1035 (size=12341) 2024-12-09T20:41:44,726 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/095424b9641342c0b43795b15ebdc0ed as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/095424b9641342c0b43795b15ebdc0ed 2024-12-09T20:41:44,743 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into 095424b9641342c0b43795b15ebdc0ed(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:44,743 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:44,743 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/56c3e1c7aaa342e382cdc6bf9fae2422 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/56c3e1c7aaa342e382cdc6bf9fae2422 2024-12-09T20:41:44,743 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=12, startTime=1733776904615; duration=0sec 2024-12-09T20:41:44,743 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:44,744 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:44,744 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:41:44,748 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:41:44,748 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:41:44,748 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:44,748 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/40010b2b3e8142dba3ce60b44e29ae0c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3d12d889cff240fcbd8b46d20fa5b61b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6e94d56fea134f23adda3bd0536a5348, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/17171fa2fc6940b9a071547cebf1ef8a] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=47.1 K 2024-12-09T20:41:44,750 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40010b2b3e8142dba3ce60b44e29ae0c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733776902520 2024-12-09T20:41:44,751 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d12d889cff240fcbd8b46d20fa5b61b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733776902610 2024-12-09T20:41:44,752 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e94d56fea134f23adda3bd0536a5348, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733776902824 2024-12-09T20:41:44,753 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17171fa2fc6940b9a071547cebf1ef8a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733776903745 2024-12-09T20:41:44,758 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into 56c3e1c7aaa342e382cdc6bf9fae2422(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:44,758 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:44,758 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=12, startTime=1733776904616; duration=0sec 2024-12-09T20:41:44,759 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:44,759 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:44,776 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:44,777 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6769edb8fd0d4b149a45b89a77bf4f3f is 50, key is test_row_0/C:col10/1733776903765/Put/seqid=0 2024-12-09T20:41:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:44,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741860_1036 (size=12341) 2024-12-09T20:41:44,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T20:41:44,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:44,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:44,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:44,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:44,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:44,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:44,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/d7787e0f7c22419eba5a44760453aae3 is 50, key is test_row_0/A:col10/1733776904792/Put/seqid=0 2024-12-09T20:41:44,826 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6769edb8fd0d4b149a45b89a77bf4f3f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6769edb8fd0d4b149a45b89a77bf4f3f 2024-12-09T20:41:44,840 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into 6769edb8fd0d4b149a45b89a77bf4f3f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:44,840 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:44,841 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=12, startTime=1733776904616; duration=0sec 2024-12-09T20:41:44,841 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:44,841 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:44,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741861_1037 (size=9757) 2024-12-09T20:41:44,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/d7787e0f7c22419eba5a44760453aae3 2024-12-09T20:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733776904838, max timestamp=1733776904860Num uniques 859; 2024-12-09T20:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733776904838, max timestamp=1733776904860Num uniques 859; 2024-12-09T20:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733776904838, max timestamp=1733776904860Num uniques 859; 2024-12-09T20:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:41:44,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/fda3af51aa554808bc02ae8e0ebf48e5 is 50, key is test_row_0/B:col10/1733776904792/Put/seqid=0 2024-12-09T20:41:44,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741862_1038 (size=9757) 2024-12-09T20:41:44,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/fda3af51aa554808bc02ae8e0ebf48e5 2024-12-09T20:41:44,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/f112d4a782a143eab1b897a5ad3fde91 is 50, key is test_row_0/C:col10/1733776904792/Put/seqid=0 2024-12-09T20:41:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776964895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776964897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:44,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776964901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:44,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776964906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:44,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776964907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:44,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741863_1039 (size=9757) 2024-12-09T20:41:45,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776965011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776965012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776965012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776965022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776965022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T20:41:45,042 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:45,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-09T20:41:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T20:41:45,048 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:45,050 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:45,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T20:41:45,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T20:41:45,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:45,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776965222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776965222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776965222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776965231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776965231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/f112d4a782a143eab1b897a5ad3fde91 2024-12-09T20:41:45,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/d7787e0f7c22419eba5a44760453aae3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d7787e0f7c22419eba5a44760453aae3 2024-12-09T20:41:45,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d7787e0f7c22419eba5a44760453aae3, entries=100, sequenceid=147, filesize=9.5 K 2024-12-09T20:41:45,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/fda3af51aa554808bc02ae8e0ebf48e5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/fda3af51aa554808bc02ae8e0ebf48e5 2024-12-09T20:41:45,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/fda3af51aa554808bc02ae8e0ebf48e5, entries=100, sequenceid=147, filesize=9.5 K 2024-12-09T20:41:45,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/f112d4a782a143eab1b897a5ad3fde91 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/f112d4a782a143eab1b897a5ad3fde91 2024-12-09T20:41:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T20:41:45,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T20:41:45,374 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T20:41:45,375 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T20:41:45,375 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T20:41:45,376 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T20:41:45,376 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T20:41:45,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T20:41:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:45,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:45,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:45,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/f112d4a782a143eab1b897a5ad3fde91, entries=100, sequenceid=147, filesize=9.5 K 2024-12-09T20:41:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:45,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dd7288626d8fbb1461f9ccf6551f1bf6 in 583ms, sequenceid=147, compaction requested=false 2024-12-09T20:41:45,381 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T20:41:45,381 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T20:41:45,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:45,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T20:41:45,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:45,533 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T20:41:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:45,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:45,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:45,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:45,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/8e51b0f29b994dcf85b830ddf3558137 is 50, key is test_row_0/A:col10/1733776905532/Put/seqid=0 2024-12-09T20:41:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776965547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776965545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776965549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776965550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776965552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741864_1040 (size=14541) 2024-12-09T20:41:45,574 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/8e51b0f29b994dcf85b830ddf3558137 2024-12-09T20:41:45,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/06acf1ecdbcb4f42a834d3984dd8082a is 50, key is test_row_0/B:col10/1733776905532/Put/seqid=0 2024-12-09T20:41:45,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741865_1041 (size=12151) 2024-12-09T20:41:45,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776965662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776965662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776965662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776965662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T20:41:45,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776965871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776965871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776965871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:45,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776965881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,028 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/06acf1ecdbcb4f42a834d3984dd8082a 2024-12-09T20:41:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/522972781da0406d92a7dd98b966ca1c is 50, key is test_row_0/C:col10/1733776905532/Put/seqid=0 2024-12-09T20:41:46,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776966062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741866_1042 (size=12151) 2024-12-09T20:41:46,106 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/522972781da0406d92a7dd98b966ca1c 2024-12-09T20:41:46,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/8e51b0f29b994dcf85b830ddf3558137 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/8e51b0f29b994dcf85b830ddf3558137 2024-12-09T20:41:46,129 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/8e51b0f29b994dcf85b830ddf3558137, entries=200, sequenceid=184, filesize=14.2 K 2024-12-09T20:41:46,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/06acf1ecdbcb4f42a834d3984dd8082a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/06acf1ecdbcb4f42a834d3984dd8082a 2024-12-09T20:41:46,149 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/06acf1ecdbcb4f42a834d3984dd8082a, entries=150, sequenceid=184, filesize=11.9 K 2024-12-09T20:41:46,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/522972781da0406d92a7dd98b966ca1c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/522972781da0406d92a7dd98b966ca1c 2024-12-09T20:41:46,169 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/522972781da0406d92a7dd98b966ca1c, entries=150, sequenceid=184, filesize=11.9 K 2024-12-09T20:41:46,172 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for dd7288626d8fbb1461f9ccf6551f1bf6 in 639ms, sequenceid=184, compaction requested=true 2024-12-09T20:41:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-09T20:41:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-09T20:41:46,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T20:41:46,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1260 sec 2024-12-09T20:41:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T20:41:46,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1380 sec 2024-12-09T20:41:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:46,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:41:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:46,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/fbf03fd4c8b74558bccd5fbc66eba4b7 is 50, key is test_row_0/A:col10/1733776905548/Put/seqid=0 2024-12-09T20:41:46,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741867_1043 (size=12151) 2024-12-09T20:41:46,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776966235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776966238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776966240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776966242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776966352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776966352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776966352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776966352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776966565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776966561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776966573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776966573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/fbf03fd4c8b74558bccd5fbc66eba4b7 2024-12-09T20:41:46,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/667320b0029146689d9e534794520ea3 is 50, key is test_row_0/B:col10/1733776905548/Put/seqid=0 2024-12-09T20:41:46,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741868_1044 (size=12151) 2024-12-09T20:41:46,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/667320b0029146689d9e534794520ea3 2024-12-09T20:41:46,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/38d8ab9363c64e828e18ef8a0d17bb8c is 50, key is test_row_0/C:col10/1733776905548/Put/seqid=0 2024-12-09T20:41:46,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741869_1045 (size=12151) 2024-12-09T20:41:46,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776966871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776966881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776966891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:46,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776966891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776967082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,087 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/38d8ab9363c64e828e18ef8a0d17bb8c 2024-12-09T20:41:47,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/fbf03fd4c8b74558bccd5fbc66eba4b7 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fbf03fd4c8b74558bccd5fbc66eba4b7 2024-12-09T20:41:47,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fbf03fd4c8b74558bccd5fbc66eba4b7, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T20:41:47,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/667320b0029146689d9e534794520ea3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/667320b0029146689d9e534794520ea3 2024-12-09T20:41:47,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/667320b0029146689d9e534794520ea3, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T20:41:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/38d8ab9363c64e828e18ef8a0d17bb8c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/38d8ab9363c64e828e18ef8a0d17bb8c 2024-12-09T20:41:47,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/38d8ab9363c64e828e18ef8a0d17bb8c, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T20:41:47,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for dd7288626d8fbb1461f9ccf6551f1bf6 in 961ms, sequenceid=197, compaction requested=true 2024-12-09T20:41:47,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:47,151 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:41:47,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:47,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:47,152 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:41:47,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:47,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:47,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:47,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:47,154 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48790 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:41:47,154 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:41:47,155 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:47,155 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/095424b9641342c0b43795b15ebdc0ed, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d7787e0f7c22419eba5a44760453aae3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/8e51b0f29b994dcf85b830ddf3558137, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fbf03fd4c8b74558bccd5fbc66eba4b7] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=47.6 K 2024-12-09T20:41:47,155 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:41:47,155 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:41:47,155 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 095424b9641342c0b43795b15ebdc0ed, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733776903745 2024-12-09T20:41:47,156 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:47,156 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/56c3e1c7aaa342e382cdc6bf9fae2422, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/fda3af51aa554808bc02ae8e0ebf48e5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/06acf1ecdbcb4f42a834d3984dd8082a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/667320b0029146689d9e534794520ea3] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=45.3 K 2024-12-09T20:41:47,156 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7787e0f7c22419eba5a44760453aae3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733776904463 2024-12-09T20:41:47,156 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 56c3e1c7aaa342e382cdc6bf9fae2422, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733776903745 2024-12-09T20:41:47,159 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e51b0f29b994dcf85b830ddf3558137, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733776904884 2024-12-09T20:41:47,159 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting fda3af51aa554808bc02ae8e0ebf48e5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733776904463 2024-12-09T20:41:47,160 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 06acf1ecdbcb4f42a834d3984dd8082a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733776904884 2024-12-09T20:41:47,163 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbf03fd4c8b74558bccd5fbc66eba4b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733776905548 2024-12-09T20:41:47,164 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 667320b0029146689d9e534794520ea3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733776905548 2024-12-09T20:41:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T20:41:47,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:47,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:47,193 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:47,194 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/4768915f1aab4a36928911f74111ccd8 is 50, key is test_row_0/B:col10/1733776905548/Put/seqid=0 2024-12-09T20:41:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-09T20:41:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T20:41:47,197 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:47,198 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:47,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:47,199 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:47,209 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/70789f81fd1a43cb82d953927a8306f9 is 50, key is test_row_0/A:col10/1733776905548/Put/seqid=0 2024-12-09T20:41:47,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741870_1046 (size=12527) 2024-12-09T20:41:47,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741871_1047 (size=12527) 2024-12-09T20:41:47,234 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/4768915f1aab4a36928911f74111ccd8 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4768915f1aab4a36928911f74111ccd8 2024-12-09T20:41:47,252 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into 4768915f1aab4a36928911f74111ccd8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:47,252 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:47,252 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=12, startTime=1733776907152; duration=0sec 2024-12-09T20:41:47,252 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:47,252 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:47,253 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:41:47,259 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:41:47,259 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:41:47,259 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:47,259 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6769edb8fd0d4b149a45b89a77bf4f3f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/f112d4a782a143eab1b897a5ad3fde91, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/522972781da0406d92a7dd98b966ca1c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/38d8ab9363c64e828e18ef8a0d17bb8c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=45.3 K 2024-12-09T20:41:47,262 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6769edb8fd0d4b149a45b89a77bf4f3f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733776903745 2024-12-09T20:41:47,263 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f112d4a782a143eab1b897a5ad3fde91, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733776904463 2024-12-09T20:41:47,264 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 522972781da0406d92a7dd98b966ca1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733776904884 2024-12-09T20:41:47,264 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d8ab9363c64e828e18ef8a0d17bb8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733776905548 2024-12-09T20:41:47,291 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:47,292 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/831c6f93a5404488a6fc51565c9bf36c is 50, key is test_row_0/C:col10/1733776905548/Put/seqid=0 2024-12-09T20:41:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T20:41:47,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741872_1048 (size=12527) 2024-12-09T20:41:47,319 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/831c6f93a5404488a6fc51565c9bf36c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/831c6f93a5404488a6fc51565c9bf36c 2024-12-09T20:41:47,335 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into 831c6f93a5404488a6fc51565c9bf36c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:47,335 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:47,335 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=12, startTime=1733776907153; duration=0sec 2024-12-09T20:41:47,335 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:47,335 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:47,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T20:41:47,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:47,363 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:41:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:47,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:47,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/a8754e0dab6146299315b3a388898d51 is 50, key is test_row_0/A:col10/1733776906235/Put/seqid=0 2024-12-09T20:41:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:47,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:47,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741873_1049 (size=12151) 2024-12-09T20:41:47,392 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/a8754e0dab6146299315b3a388898d51 2024-12-09T20:41:47,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/d6d94f2a8cef4ae5b6eb27578b533dee is 50, key is test_row_0/B:col10/1733776906235/Put/seqid=0 2024-12-09T20:41:47,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776967411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776967416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776967414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776967416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741874_1050 (size=12151) 2024-12-09T20:41:47,436 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/d6d94f2a8cef4ae5b6eb27578b533dee 2024-12-09T20:41:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/9f628ba6656942feb71bcec8c6f78469 is 50, key is test_row_0/C:col10/1733776906235/Put/seqid=0 2024-12-09T20:41:47,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741875_1051 (size=12151) 2024-12-09T20:41:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T20:41:47,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776967531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776967531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776967531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,635 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/70789f81fd1a43cb82d953927a8306f9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/70789f81fd1a43cb82d953927a8306f9 2024-12-09T20:41:47,643 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into 70789f81fd1a43cb82d953927a8306f9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:47,643 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:47,643 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=12, startTime=1733776907151; duration=0sec 2024-12-09T20:41:47,644 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:47,644 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776967741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776967741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776967741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T20:41:47,863 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/9f628ba6656942feb71bcec8c6f78469 2024-12-09T20:41:47,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/a8754e0dab6146299315b3a388898d51 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a8754e0dab6146299315b3a388898d51 2024-12-09T20:41:47,879 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a8754e0dab6146299315b3a388898d51, entries=150, sequenceid=218, filesize=11.9 K 2024-12-09T20:41:47,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/d6d94f2a8cef4ae5b6eb27578b533dee as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d6d94f2a8cef4ae5b6eb27578b533dee 2024-12-09T20:41:47,890 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d6d94f2a8cef4ae5b6eb27578b533dee, entries=150, sequenceid=218, filesize=11.9 K 2024-12-09T20:41:47,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/9f628ba6656942feb71bcec8c6f78469 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/9f628ba6656942feb71bcec8c6f78469 2024-12-09T20:41:47,902 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/9f628ba6656942feb71bcec8c6f78469, entries=150, sequenceid=218, filesize=11.9 K 2024-12-09T20:41:47,903 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for dd7288626d8fbb1461f9ccf6551f1bf6 in 540ms, sequenceid=218, compaction requested=false 2024-12-09T20:41:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-09T20:41:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-09T20:41:47,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T20:41:47,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 707 msec 2024-12-09T20:41:47,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 715 msec 2024-12-09T20:41:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:48,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T20:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:48,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/010555646c0146f09ce60e71364a6262 is 50, key is test_row_0/A:col10/1733776907413/Put/seqid=0 2024-12-09T20:41:48,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741876_1052 (size=12151) 2024-12-09T20:41:48,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776968085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776968091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776968092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776968202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776968202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776968202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T20:41:48,332 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:48,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-09T20:41:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T20:41:48,336 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:48,338 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:48,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776968411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776968411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776968412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776968422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,424 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T20:41:48,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/010555646c0146f09ce60e71364a6262 2024-12-09T20:41:48,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T20:41:48,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/6b9d1b46bb2347b99dfee8b17a75d3ae is 50, key is test_row_0/B:col10/1733776907413/Put/seqid=0 2024-12-09T20:41:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:48,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:48,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:48,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741877_1053 (size=12151) 2024-12-09T20:41:48,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/6b9d1b46bb2347b99dfee8b17a75d3ae 2024-12-09T20:41:48,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6f6e560424a94cbab3e3903c42565186 is 50, key is test_row_0/C:col10/1733776907413/Put/seqid=0 2024-12-09T20:41:48,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741878_1054 (size=12151) 2024-12-09T20:41:48,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6f6e560424a94cbab3e3903c42565186 2024-12-09T20:41:48,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/010555646c0146f09ce60e71364a6262 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/010555646c0146f09ce60e71364a6262 2024-12-09T20:41:48,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/010555646c0146f09ce60e71364a6262, entries=150, sequenceid=235, filesize=11.9 K 2024-12-09T20:41:48,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/6b9d1b46bb2347b99dfee8b17a75d3ae as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6b9d1b46bb2347b99dfee8b17a75d3ae 2024-12-09T20:41:48,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6b9d1b46bb2347b99dfee8b17a75d3ae, entries=150, sequenceid=235, filesize=11.9 K 2024-12-09T20:41:48,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6f6e560424a94cbab3e3903c42565186 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6f6e560424a94cbab3e3903c42565186 2024-12-09T20:41:48,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6f6e560424a94cbab3e3903c42565186, entries=150, sequenceid=235, filesize=11.9 K 2024-12-09T20:41:48,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for dd7288626d8fbb1461f9ccf6551f1bf6 in 526ms, sequenceid=235, compaction requested=true 2024-12-09T20:41:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:48,584 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:48,584 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:48,585 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:48,586 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:41:48,586 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:48,586 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/70789f81fd1a43cb82d953927a8306f9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a8754e0dab6146299315b3a388898d51, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/010555646c0146f09ce60e71364a6262] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.0 K 2024-12-09T20:41:48,586 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:48,586 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:41:48,586 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:48,586 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70789f81fd1a43cb82d953927a8306f9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733776905548 2024-12-09T20:41:48,587 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4768915f1aab4a36928911f74111ccd8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d6d94f2a8cef4ae5b6eb27578b533dee, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6b9d1b46bb2347b99dfee8b17a75d3ae] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.0 K 2024-12-09T20:41:48,587 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8754e0dab6146299315b3a388898d51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733776906235 2024-12-09T20:41:48,587 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 4768915f1aab4a36928911f74111ccd8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733776905548 2024-12-09T20:41:48,588 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d6d94f2a8cef4ae5b6eb27578b533dee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733776906235 2024-12-09T20:41:48,588 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 010555646c0146f09ce60e71364a6262, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733776907413 2024-12-09T20:41:48,589 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b9d1b46bb2347b99dfee8b17a75d3ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733776907413 2024-12-09T20:41:48,607 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:48,608 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/b1f37ee4a162419ea7498258302ef459 is 50, key is test_row_0/B:col10/1733776907413/Put/seqid=0 2024-12-09T20:41:48,619 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:48,620 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/c78edf52077748c5a7e7b7c52af5f5c2 is 50, key is test_row_0/A:col10/1733776907413/Put/seqid=0 2024-12-09T20:41:48,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741880_1056 (size=12629) 2024-12-09T20:41:48,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741879_1055 (size=12629) 2024-12-09T20:41:48,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T20:41:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:48,648 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T20:41:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:48,649 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/c78edf52077748c5a7e7b7c52af5f5c2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c78edf52077748c5a7e7b7c52af5f5c2 2024-12-09T20:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T20:41:48,656 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/b1f37ee4a162419ea7498258302ef459 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b1f37ee4a162419ea7498258302ef459 2024-12-09T20:41:48,663 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into c78edf52077748c5a7e7b7c52af5f5c2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:48,663 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:48,663 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=13, startTime=1733776908584; duration=0sec 2024-12-09T20:41:48,663 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:48,663 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:48,663 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:48,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:48,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:41:48,666 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:48,667 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/831c6f93a5404488a6fc51565c9bf36c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/9f628ba6656942feb71bcec8c6f78469, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6f6e560424a94cbab3e3903c42565186] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.0 K 2024-12-09T20:41:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/21192a7361af434aa8429354f5607ff2 is 50, key is test_row_0/A:col10/1733776908083/Put/seqid=0 2024-12-09T20:41:48,669 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 831c6f93a5404488a6fc51565c9bf36c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733776905548 2024-12-09T20:41:48,670 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f628ba6656942feb71bcec8c6f78469, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733776906235 2024-12-09T20:41:48,670 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f6e560424a94cbab3e3903c42565186, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733776907413 2024-12-09T20:41:48,671 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into b1f37ee4a162419ea7498258302ef459(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:48,671 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:48,671 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=13, startTime=1733776908584; duration=0sec 2024-12-09T20:41:48,671 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:48,672 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:48,686 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:48,687 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/0f82018aa7da41d2b71171febc0adadc is 50, key is test_row_0/C:col10/1733776907413/Put/seqid=0 2024-12-09T20:41:48,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741881_1057 (size=12151) 2024-12-09T20:41:48,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741882_1058 (size=12629) 2024-12-09T20:41:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:48,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:48,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776968748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776968754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776968760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776968861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776968871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776968872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T20:41:49,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776969071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776969081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776969081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776969101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,103 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,113 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/21192a7361af434aa8429354f5607ff2 2024-12-09T20:41:49,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/b53bbc5f53a7428bb3ad2336f57dbcb5 is 50, key is test_row_0/B:col10/1733776908083/Put/seqid=0 2024-12-09T20:41:49,136 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/0f82018aa7da41d2b71171febc0adadc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0f82018aa7da41d2b71171febc0adadc 2024-12-09T20:41:49,145 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into 0f82018aa7da41d2b71171febc0adadc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:49,145 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:49,145 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=13, startTime=1733776908584; duration=0sec 2024-12-09T20:41:49,145 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:49,145 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:49,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741883_1059 (size=12151) 2024-12-09T20:41:49,149 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/b53bbc5f53a7428bb3ad2336f57dbcb5 2024-12-09T20:41:49,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/b8f008c5bc2641158d29a43e9576df0a is 50, key is test_row_0/C:col10/1733776908083/Put/seqid=0 2024-12-09T20:41:49,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741884_1060 (size=12151) 2024-12-09T20:41:49,175 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/b8f008c5bc2641158d29a43e9576df0a 2024-12-09T20:41:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/21192a7361af434aa8429354f5607ff2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/21192a7361af434aa8429354f5607ff2 2024-12-09T20:41:49,192 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/21192a7361af434aa8429354f5607ff2, entries=150, sequenceid=252, filesize=11.9 K 2024-12-09T20:41:49,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/b53bbc5f53a7428bb3ad2336f57dbcb5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b53bbc5f53a7428bb3ad2336f57dbcb5 2024-12-09T20:41:49,241 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b53bbc5f53a7428bb3ad2336f57dbcb5, entries=150, sequenceid=252, filesize=11.9 K 2024-12-09T20:41:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/b8f008c5bc2641158d29a43e9576df0a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/b8f008c5bc2641158d29a43e9576df0a 2024-12-09T20:41:49,248 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/b8f008c5bc2641158d29a43e9576df0a, entries=150, sequenceid=252, filesize=11.9 K 2024-12-09T20:41:49,250 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for dd7288626d8fbb1461f9ccf6551f1bf6 in 601ms, sequenceid=252, compaction requested=false 2024-12-09T20:41:49,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:49,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:49,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-09T20:41:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-09T20:41:49,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-09T20:41:49,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 914 msec 2024-12-09T20:41:49,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 923 msec 2024-12-09T20:41:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:49,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T20:41:49,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:49,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:49,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:49,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:49,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:49,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:49,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/97a354e3cf6c45ee915765075ea08e7c is 50, key is test_row_0/A:col10/1733776908743/Put/seqid=0 2024-12-09T20:41:49,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741885_1061 (size=12301) 2024-12-09T20:41:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T20:41:49,472 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:49,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-09T20:41:49,476 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:49,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776969471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T20:41:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,478 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:49,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:49,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776969473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776969478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T20:41:49,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776969581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776969591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776969591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T20:41:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:49,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:49,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T20:41:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:49,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T20:41:49,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776969791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776969801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776969801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:49,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/97a354e3cf6c45ee915765075ea08e7c 2024-12-09T20:41:49,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/9fd0d7d2ae6446e8a4a0d62bd9f4a132 is 50, key is test_row_0/B:col10/1733776908743/Put/seqid=0 2024-12-09T20:41:49,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741886_1062 (size=12301) 2024-12-09T20:41:49,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T20:41:49,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:49,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:49,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:50,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T20:41:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:50,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T20:41:50,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776970102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:50,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776970111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:50,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776970111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:50,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/9fd0d7d2ae6446e8a4a0d62bd9f4a132 2024-12-09T20:41:50,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/111caa6f822a47cebe3c6082a947cd97 is 50, key is test_row_0/C:col10/1733776908743/Put/seqid=0 2024-12-09T20:41:50,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741887_1063 (size=12301) 2024-12-09T20:41:50,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/111caa6f822a47cebe3c6082a947cd97 2024-12-09T20:41:50,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T20:41:50,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:50,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:50,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:50,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:50,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:50,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:50,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/97a354e3cf6c45ee915765075ea08e7c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/97a354e3cf6c45ee915765075ea08e7c 2024-12-09T20:41:50,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/97a354e3cf6c45ee915765075ea08e7c, entries=150, sequenceid=271, filesize=12.0 K 2024-12-09T20:41:50,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/9fd0d7d2ae6446e8a4a0d62bd9f4a132 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/9fd0d7d2ae6446e8a4a0d62bd9f4a132 2024-12-09T20:41:50,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/9fd0d7d2ae6446e8a4a0d62bd9f4a132, entries=150, sequenceid=271, filesize=12.0 K 2024-12-09T20:41:50,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/111caa6f822a47cebe3c6082a947cd97 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/111caa6f822a47cebe3c6082a947cd97 2024-12-09T20:41:50,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/111caa6f822a47cebe3c6082a947cd97, entries=150, sequenceid=271, filesize=12.0 K 2024-12-09T20:41:50,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for dd7288626d8fbb1461f9ccf6551f1bf6 in 913ms, sequenceid=271, compaction requested=true 2024-12-09T20:41:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:50,298 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:50,298 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:50,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:50,300 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:50,300 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:50,300 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:41:50,300 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:41:50,300 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:50,300 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:50,301 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b1f37ee4a162419ea7498258302ef459, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b53bbc5f53a7428bb3ad2336f57dbcb5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/9fd0d7d2ae6446e8a4a0d62bd9f4a132] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.2 K 2024-12-09T20:41:50,301 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c78edf52077748c5a7e7b7c52af5f5c2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/21192a7361af434aa8429354f5607ff2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/97a354e3cf6c45ee915765075ea08e7c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.2 K 2024-12-09T20:41:50,301 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting c78edf52077748c5a7e7b7c52af5f5c2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733776907413 2024-12-09T20:41:50,301 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f37ee4a162419ea7498258302ef459, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733776907413 2024-12-09T20:41:50,305 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21192a7361af434aa8429354f5607ff2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733776908083 2024-12-09T20:41:50,305 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b53bbc5f53a7428bb3ad2336f57dbcb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733776908083 2024-12-09T20:41:50,307 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97a354e3cf6c45ee915765075ea08e7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733776908743 2024-12-09T20:41:50,308 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fd0d7d2ae6446e8a4a0d62bd9f4a132, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733776908743 2024-12-09T20:41:50,322 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:50,323 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/10ba878fee7d423d91f38e13673f7ad6 is 50, key is test_row_0/A:col10/1733776908743/Put/seqid=0 2024-12-09T20:41:50,331 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:50,332 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/54411d0f008b40cda8d178f692e841a6 is 50, key is test_row_0/B:col10/1733776908743/Put/seqid=0 2024-12-09T20:41:50,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741888_1064 (size=12881) 2024-12-09T20:41:50,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741889_1065 (size=12881) 2024-12-09T20:41:50,377 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/54411d0f008b40cda8d178f692e841a6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/54411d0f008b40cda8d178f692e841a6 2024-12-09T20:41:50,384 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/10ba878fee7d423d91f38e13673f7ad6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/10ba878fee7d423d91f38e13673f7ad6 2024-12-09T20:41:50,395 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into 54411d0f008b40cda8d178f692e841a6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:50,395 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:50,395 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=13, startTime=1733776910298; duration=0sec 2024-12-09T20:41:50,396 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:50,396 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:50,396 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:50,399 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:50,399 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:41:50,400 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:50,400 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0f82018aa7da41d2b71171febc0adadc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/b8f008c5bc2641158d29a43e9576df0a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/111caa6f822a47cebe3c6082a947cd97] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.2 K 2024-12-09T20:41:50,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T20:41:50,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:50,401 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f82018aa7da41d2b71171febc0adadc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733776907413 2024-12-09T20:41:50,401 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T20:41:50,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:50,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:50,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:50,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:50,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:50,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:50,402 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f008c5bc2641158d29a43e9576df0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733776908083 2024-12-09T20:41:50,403 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 111caa6f822a47cebe3c6082a947cd97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733776908743 2024-12-09T20:41:50,405 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into 10ba878fee7d423d91f38e13673f7ad6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:50,405 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:50,405 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=13, startTime=1733776910298; duration=0sec 2024-12-09T20:41:50,405 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:50,406 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:50,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/e29505a12a7a4b2ea2350da0f2628b68 is 50, key is test_row_0/A:col10/1733776909474/Put/seqid=0 2024-12-09T20:41:50,428 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:50,428 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/8c38fc7255774ae0be9ce8fb232a787e is 50, key is test_row_0/C:col10/1733776908743/Put/seqid=0 2024-12-09T20:41:50,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741890_1066 (size=12301) 2024-12-09T20:41:50,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:50,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741891_1067 (size=12881) 2024-12-09T20:41:50,470 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/8c38fc7255774ae0be9ce8fb232a787e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/8c38fc7255774ae0be9ce8fb232a787e 2024-12-09T20:41:50,479 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into 8c38fc7255774ae0be9ce8fb232a787e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:50,479 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:50,479 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=13, startTime=1733776910298; duration=0sec 2024-12-09T20:41:50,479 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:50,480 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:50,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776970492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:50,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776970601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T20:41:50,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776970611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:50,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:50,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776970622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776970622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:50,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776970812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:50,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:50,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:50,841 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/e29505a12a7a4b2ea2350da0f2628b68 2024-12-09T20:41:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/4037069c058645fb946a355d66548fdf is 50, key is test_row_0/B:col10/1733776909474/Put/seqid=0 2024-12-09T20:41:50,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741892_1068 (size=12301) 2024-12-09T20:41:50,860 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/4037069c058645fb946a355d66548fdf 2024-12-09T20:41:50,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6c68a6bba0f04485b7915e39848669a2 is 50, key is test_row_0/C:col10/1733776909474/Put/seqid=0 2024-12-09T20:41:50,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741893_1069 (size=12301) 2024-12-09T20:41:51,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776971121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,275 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6c68a6bba0f04485b7915e39848669a2 2024-12-09T20:41:51,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/e29505a12a7a4b2ea2350da0f2628b68 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e29505a12a7a4b2ea2350da0f2628b68 2024-12-09T20:41:51,289 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e29505a12a7a4b2ea2350da0f2628b68, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T20:41:51,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/4037069c058645fb946a355d66548fdf as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4037069c058645fb946a355d66548fdf 2024-12-09T20:41:51,296 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4037069c058645fb946a355d66548fdf, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T20:41:51,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6c68a6bba0f04485b7915e39848669a2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6c68a6bba0f04485b7915e39848669a2 2024-12-09T20:41:51,305 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6c68a6bba0f04485b7915e39848669a2, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T20:41:51,306 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for dd7288626d8fbb1461f9ccf6551f1bf6 in 906ms, sequenceid=288, compaction requested=false 2024-12-09T20:41:51,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:51,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:51,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-09T20:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-09T20:41:51,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T20:41:51,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-12-09T20:41:51,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8410 sec 2024-12-09T20:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T20:41:51,621 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:51,622 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-09T20:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:41:51,624 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:51,625 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:51,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:51,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T20:41:51,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:51,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:51,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:51,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:51,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:51,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:51,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/b6717ace514e4578a935021b04d3a27f is 50, key is test_row_0/A:col10/1733776911633/Put/seqid=0 2024-12-09T20:41:51,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741894_1070 (size=14741) 2024-12-09T20:41:51,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776971674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776971674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776971674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776971675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:41:51,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T20:41:51,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:51,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:51,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:51,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:51,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776971781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776971781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776971791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776971791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T20:41:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:51,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:41:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776971992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776971992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776972001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776972001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/b6717ace514e4578a935021b04d3a27f 2024-12-09T20:41:52,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/d92f3e3430ea429390768fab3bff6af2 is 50, key is test_row_0/B:col10/1733776911633/Put/seqid=0 2024-12-09T20:41:52,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741895_1071 (size=12301) 2024-12-09T20:41:52,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T20:41:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T20:41:52,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:52,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:41:52,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776972301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776972302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776972311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776972311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T20:41:52,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/d92f3e3430ea429390768fab3bff6af2 2024-12-09T20:41:52,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/83d0549651cd4e1ab1737d0712ced939 is 50, key is test_row_0/C:col10/1733776911633/Put/seqid=0 2024-12-09T20:41:52,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741896_1072 (size=12301) 2024-12-09T20:41:52,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T20:41:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T20:41:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:41:52,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776972811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776972811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776972821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776972821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:52,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T20:41:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:52,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/83d0549651cd4e1ab1737d0712ced939 2024-12-09T20:41:52,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/b6717ace514e4578a935021b04d3a27f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/b6717ace514e4578a935021b04d3a27f 2024-12-09T20:41:52,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/b6717ace514e4578a935021b04d3a27f, entries=200, sequenceid=309, filesize=14.4 K 2024-12-09T20:41:52,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/d92f3e3430ea429390768fab3bff6af2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d92f3e3430ea429390768fab3bff6af2 2024-12-09T20:41:52,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d92f3e3430ea429390768fab3bff6af2, entries=150, sequenceid=309, filesize=12.0 K 2024-12-09T20:41:52,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/83d0549651cd4e1ab1737d0712ced939 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/83d0549651cd4e1ab1737d0712ced939 2024-12-09T20:41:52,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/83d0549651cd4e1ab1737d0712ced939, entries=150, sequenceid=309, filesize=12.0 K 2024-12-09T20:41:52,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for dd7288626d8fbb1461f9ccf6551f1bf6 in 1285ms, sequenceid=309, compaction requested=true 2024-12-09T20:41:52,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:52,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:52,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:52,923 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:52,923 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:52,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:52,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:52,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:52,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:52,927 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:52,927 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:41:52,927 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,928 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/10ba878fee7d423d91f38e13673f7ad6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e29505a12a7a4b2ea2350da0f2628b68, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/b6717ace514e4578a935021b04d3a27f] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=39.0 K 2024-12-09T20:41:52,928 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:52,928 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:41:52,928 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,928 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/54411d0f008b40cda8d178f692e841a6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4037069c058645fb946a355d66548fdf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d92f3e3430ea429390768fab3bff6af2] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.6 K 2024-12-09T20:41:52,929 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10ba878fee7d423d91f38e13673f7ad6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733776908743 2024-12-09T20:41:52,929 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 54411d0f008b40cda8d178f692e841a6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733776908743 2024-12-09T20:41:52,929 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e29505a12a7a4b2ea2350da0f2628b68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733776909406 2024-12-09T20:41:52,930 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 4037069c058645fb946a355d66548fdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733776909406 2024-12-09T20:41:52,930 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6717ace514e4578a935021b04d3a27f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733776910464 2024-12-09T20:41:52,930 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d92f3e3430ea429390768fab3bff6af2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733776910488 2024-12-09T20:41:52,941 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:52,942 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/76151ea923c642ce93dec8adb97b9cbb is 50, key is test_row_0/B:col10/1733776911633/Put/seqid=0 2024-12-09T20:41:52,950 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:52,951 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/023157ca7d6e41e28b68e6a183182d83 is 50, key is test_row_0/A:col10/1733776911633/Put/seqid=0 2024-12-09T20:41:52,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741897_1073 (size=12983) 2024-12-09T20:41:52,975 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/76151ea923c642ce93dec8adb97b9cbb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/76151ea923c642ce93dec8adb97b9cbb 2024-12-09T20:41:52,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741898_1074 (size=12983) 2024-12-09T20:41:52,988 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into 76151ea923c642ce93dec8adb97b9cbb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:52,988 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:52,989 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=13, startTime=1733776912923; duration=0sec 2024-12-09T20:41:52,989 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:52,989 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:52,990 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:52,991 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/023157ca7d6e41e28b68e6a183182d83 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/023157ca7d6e41e28b68e6a183182d83 2024-12-09T20:41:52,992 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:52,992 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:41:52,992 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:52,993 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/8c38fc7255774ae0be9ce8fb232a787e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6c68a6bba0f04485b7915e39848669a2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/83d0549651cd4e1ab1737d0712ced939] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.6 K 2024-12-09T20:41:52,994 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c38fc7255774ae0be9ce8fb232a787e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733776908743 2024-12-09T20:41:52,996 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c68a6bba0f04485b7915e39848669a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733776909406 2024-12-09T20:41:52,997 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d0549651cd4e1ab1737d0712ced939, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733776910488 2024-12-09T20:41:53,003 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into 023157ca7d6e41e28b68e6a183182d83(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:53,003 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:53,003 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=13, startTime=1733776912923; duration=0sec 2024-12-09T20:41:53,005 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:53,005 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:53,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T20:41:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:53,008 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:41:53,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:53,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:53,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:53,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:53,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:53,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/04485857216246abb4de5b4403a57115 is 50, key is test_row_0/A:col10/1733776911673/Put/seqid=0 2024-12-09T20:41:53,018 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:53,019 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/73a12cc3e81c498dbebc9f5c24a35e68 is 50, key is test_row_0/C:col10/1733776911633/Put/seqid=0 2024-12-09T20:41:53,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741899_1075 (size=12301) 2024-12-09T20:41:53,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741900_1076 (size=12983) 2024-12-09T20:41:53,043 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/73a12cc3e81c498dbebc9f5c24a35e68 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73a12cc3e81c498dbebc9f5c24a35e68 2024-12-09T20:41:53,050 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into 73a12cc3e81c498dbebc9f5c24a35e68(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:53,050 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:53,051 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=13, startTime=1733776912924; duration=0sec 2024-12-09T20:41:53,051 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:53,051 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:53,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:53,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776973199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:53,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776973312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:53,430 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/04485857216246abb4de5b4403a57115 2024-12-09T20:41:53,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/ba901999a91e4146a5aa45e138175dbb is 50, key is test_row_0/B:col10/1733776911673/Put/seqid=0 2024-12-09T20:41:53,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741901_1077 (size=12301) 2024-12-09T20:41:53,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776973521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:41:53,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776973821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:53,822 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:53,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776973831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:53,833 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:53,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776973832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:53,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776973832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:53,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776973832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:53,836 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:53,837 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:53,855 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/ba901999a91e4146a5aa45e138175dbb 2024-12-09T20:41:53,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/86cd147c27224d938b5f90b6aa3be395 is 50, key is test_row_0/C:col10/1733776911673/Put/seqid=0 2024-12-09T20:41:53,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741902_1078 (size=12301) 2024-12-09T20:41:53,897 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/86cd147c27224d938b5f90b6aa3be395 2024-12-09T20:41:53,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/04485857216246abb4de5b4403a57115 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/04485857216246abb4de5b4403a57115 2024-12-09T20:41:53,914 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/04485857216246abb4de5b4403a57115, entries=150, sequenceid=324, filesize=12.0 K 2024-12-09T20:41:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/ba901999a91e4146a5aa45e138175dbb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/ba901999a91e4146a5aa45e138175dbb 2024-12-09T20:41:53,922 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/ba901999a91e4146a5aa45e138175dbb, entries=150, sequenceid=324, filesize=12.0 K 2024-12-09T20:41:53,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/86cd147c27224d938b5f90b6aa3be395 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/86cd147c27224d938b5f90b6aa3be395 2024-12-09T20:41:53,932 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/86cd147c27224d938b5f90b6aa3be395, entries=150, sequenceid=324, filesize=12.0 K 2024-12-09T20:41:53,939 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for dd7288626d8fbb1461f9ccf6551f1bf6 in 930ms, sequenceid=324, compaction requested=false 2024-12-09T20:41:53,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:53,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:53,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-09T20:41:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-09T20:41:53,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T20:41:53,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-12-09T20:41:53,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.3330 sec 2024-12-09T20:41:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:54,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:41:54,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:54,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:54,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:54,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:54,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:54,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:54,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776974389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:54,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/74a340fae40f4e189cfe77bf9618ff39 is 50, key is test_row_0/A:col10/1733776913171/Put/seqid=0 2024-12-09T20:41:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:54,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741903_1079 (size=14741) 2024-12-09T20:41:54,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776974501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:54,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:54,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:54,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776974711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:54,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/74a340fae40f4e189cfe77bf9618ff39 2024-12-09T20:41:54,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/5e994942c78b46e2962039bbad2fb9e8 is 50, key is test_row_0/B:col10/1733776913171/Put/seqid=0 2024-12-09T20:41:54,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741904_1080 (size=12301) 2024-12-09T20:41:54,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/5e994942c78b46e2962039bbad2fb9e8 2024-12-09T20:41:54,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2e7817f247fc451086c13f0b87a6d26c is 50, key is test_row_0/C:col10/1733776913171/Put/seqid=0 2024-12-09T20:41:54,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741905_1081 (size=12301) 2024-12-09T20:41:54,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2e7817f247fc451086c13f0b87a6d26c 2024-12-09T20:41:54,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/74a340fae40f4e189cfe77bf9618ff39 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/74a340fae40f4e189cfe77bf9618ff39 2024-12-09T20:41:54,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/74a340fae40f4e189cfe77bf9618ff39, entries=200, sequenceid=345, filesize=14.4 K 2024-12-09T20:41:54,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/5e994942c78b46e2962039bbad2fb9e8 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/5e994942c78b46e2962039bbad2fb9e8 2024-12-09T20:41:54,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/5e994942c78b46e2962039bbad2fb9e8, entries=150, sequenceid=345, filesize=12.0 K 2024-12-09T20:41:54,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2e7817f247fc451086c13f0b87a6d26c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2e7817f247fc451086c13f0b87a6d26c 2024-12-09T20:41:54,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2e7817f247fc451086c13f0b87a6d26c, entries=150, sequenceid=345, filesize=12.0 K 2024-12-09T20:41:54,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for dd7288626d8fbb1461f9ccf6551f1bf6 in 544ms, sequenceid=345, compaction requested=true 2024-12-09T20:41:54,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:54,890 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:54,890 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:54,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:54,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:54,891 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:54,891 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:41:54,891 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:54,892 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/023157ca7d6e41e28b68e6a183182d83, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/04485857216246abb4de5b4403a57115, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/74a340fae40f4e189cfe77bf9618ff39] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=39.1 K 2024-12-09T20:41:54,893 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:54,894 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:41:54,894 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:54,894 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/76151ea923c642ce93dec8adb97b9cbb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/ba901999a91e4146a5aa45e138175dbb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/5e994942c78b46e2962039bbad2fb9e8] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.7 K 2024-12-09T20:41:54,894 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 023157ca7d6e41e28b68e6a183182d83, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733776910488 2024-12-09T20:41:54,894 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 76151ea923c642ce93dec8adb97b9cbb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733776910488 2024-12-09T20:41:54,895 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting ba901999a91e4146a5aa45e138175dbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733776911647 2024-12-09T20:41:54,895 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04485857216246abb4de5b4403a57115, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733776911647 2024-12-09T20:41:54,895 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a340fae40f4e189cfe77bf9618ff39, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733776913171 2024-12-09T20:41:54,895 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e994942c78b46e2962039bbad2fb9e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733776913171 2024-12-09T20:41:54,911 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:54,911 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/b48f1bb7667e4b9f953b73955d804a05 is 50, key is test_row_0/B:col10/1733776913171/Put/seqid=0 2024-12-09T20:41:54,916 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:54,916 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/e8c1ea7b957c49ad80d56a937bff1d53 is 50, key is test_row_0/A:col10/1733776913171/Put/seqid=0 2024-12-09T20:41:54,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741907_1083 (size=13085) 2024-12-09T20:41:54,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741906_1082 (size=13085) 2024-12-09T20:41:54,939 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/e8c1ea7b957c49ad80d56a937bff1d53 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e8c1ea7b957c49ad80d56a937bff1d53 2024-12-09T20:41:54,939 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/b48f1bb7667e4b9f953b73955d804a05 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b48f1bb7667e4b9f953b73955d804a05 2024-12-09T20:41:54,946 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into b48f1bb7667e4b9f953b73955d804a05(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:54,946 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:54,946 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=13, startTime=1733776914890; duration=0sec 2024-12-09T20:41:54,946 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into e8c1ea7b957c49ad80d56a937bff1d53(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:54,946 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:54,946 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:54,946 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=13, startTime=1733776914890; duration=0sec 2024-12-09T20:41:54,946 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:54,947 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:54,947 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:54,947 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:54,950 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:54,950 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:41:54,950 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:54,950 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73a12cc3e81c498dbebc9f5c24a35e68, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/86cd147c27224d938b5f90b6aa3be395, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2e7817f247fc451086c13f0b87a6d26c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.7 K 2024-12-09T20:41:54,951 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 73a12cc3e81c498dbebc9f5c24a35e68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733776910488 2024-12-09T20:41:54,951 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 86cd147c27224d938b5f90b6aa3be395, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733776911647 2024-12-09T20:41:54,951 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e7817f247fc451086c13f0b87a6d26c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733776913171 2024-12-09T20:41:54,959 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:54,960 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6d829b2770654964a27f54083c26ddf2 is 50, key is test_row_0/C:col10/1733776913171/Put/seqid=0 2024-12-09T20:41:54,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741908_1084 (size=13085) 2024-12-09T20:41:54,976 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6d829b2770654964a27f54083c26ddf2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6d829b2770654964a27f54083c26ddf2 2024-12-09T20:41:54,983 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into 6d829b2770654964a27f54083c26ddf2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:54,983 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:54,983 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=13, startTime=1733776914891; duration=0sec 2024-12-09T20:41:54,983 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:54,983 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:55,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:41:55,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:55,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:55,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:55,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:55,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:55,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:55,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/3255840b9f70453f9cd1d97a78bec6e9 is 50, key is test_row_0/A:col10/1733776915026/Put/seqid=0 2024-12-09T20:41:55,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741909_1085 (size=12301) 2024-12-09T20:41:55,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776975073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:55,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776975181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:55,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776975391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:55,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/3255840b9f70453f9cd1d97a78bec6e9 2024-12-09T20:41:55,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0364fd5255384d49a93d7e53f53c8c59 is 50, key is test_row_0/B:col10/1733776915026/Put/seqid=0 2024-12-09T20:41:55,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741910_1086 (size=12301) 2024-12-09T20:41:55,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776975701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:41:55,791 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:55,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-09T20:41:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T20:41:55,795 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:55,795 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:55,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:55,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776975830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:55,833 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:55,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776975851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:55,853 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:55,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:55,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776975861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:55,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776975862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:55,862 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:55,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0364fd5255384d49a93d7e53f53c8c59 2024-12-09T20:41:55,864 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:55,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2409f680da3b4f679903f2552f492e3f is 50, key is test_row_0/C:col10/1733776915026/Put/seqid=0 2024-12-09T20:41:55,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741911_1087 (size=12301) 2024-12-09T20:41:55,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2409f680da3b4f679903f2552f492e3f 2024-12-09T20:41:55,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/3255840b9f70453f9cd1d97a78bec6e9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/3255840b9f70453f9cd1d97a78bec6e9 2024-12-09T20:41:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T20:41:55,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/3255840b9f70453f9cd1d97a78bec6e9, entries=150, sequenceid=361, filesize=12.0 K 2024-12-09T20:41:55,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0364fd5255384d49a93d7e53f53c8c59 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0364fd5255384d49a93d7e53f53c8c59 2024-12-09T20:41:55,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0364fd5255384d49a93d7e53f53c8c59, entries=150, sequenceid=361, filesize=12.0 K 2024-12-09T20:41:55,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2409f680da3b4f679903f2552f492e3f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2409f680da3b4f679903f2552f492e3f 2024-12-09T20:41:55,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2409f680da3b4f679903f2552f492e3f, entries=150, sequenceid=361, filesize=12.0 K 2024-12-09T20:41:55,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for dd7288626d8fbb1461f9ccf6551f1bf6 in 896ms, sequenceid=361, compaction requested=false 2024-12-09T20:41:55,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:55,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T20:41:55,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:55,948 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:41:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:55,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/c718c9f01f8e4c9f8acfc668963c1755 is 50, key is test_row_0/A:col10/1733776915063/Put/seqid=0 2024-12-09T20:41:55,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741912_1088 (size=12301) 2024-12-09T20:41:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T20:41:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:56,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:56,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776976259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:56,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776976371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:56,372 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/c718c9f01f8e4c9f8acfc668963c1755 2024-12-09T20:41:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:56,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/bd2e936328ac4f95a59d53e83b1b724f is 50, key is test_row_0/B:col10/1733776915063/Put/seqid=0 2024-12-09T20:41:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741913_1089 (size=12301) 2024-12-09T20:41:56,397 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/bd2e936328ac4f95a59d53e83b1b724f 2024-12-09T20:41:56,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/a51edf690b2a427da34f121cf8ba6bdf is 50, key is test_row_0/C:col10/1733776915063/Put/seqid=0 2024-12-09T20:41:56,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741914_1090 (size=12301) 2024-12-09T20:41:56,411 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/a51edf690b2a427da34f121cf8ba6bdf 2024-12-09T20:41:56,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/c718c9f01f8e4c9f8acfc668963c1755 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c718c9f01f8e4c9f8acfc668963c1755 2024-12-09T20:41:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T20:41:56,422 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c718c9f01f8e4c9f8acfc668963c1755, entries=150, sequenceid=380, filesize=12.0 K 2024-12-09T20:41:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/bd2e936328ac4f95a59d53e83b1b724f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/bd2e936328ac4f95a59d53e83b1b724f 2024-12-09T20:41:56,431 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/bd2e936328ac4f95a59d53e83b1b724f, entries=150, sequenceid=380, filesize=12.0 K 2024-12-09T20:41:56,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/a51edf690b2a427da34f121cf8ba6bdf as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/a51edf690b2a427da34f121cf8ba6bdf 2024-12-09T20:41:56,439 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/a51edf690b2a427da34f121cf8ba6bdf, entries=150, sequenceid=380, filesize=12.0 K 2024-12-09T20:41:56,440 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for dd7288626d8fbb1461f9ccf6551f1bf6 in 492ms, sequenceid=380, compaction requested=true 2024-12-09T20:41:56,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:56,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:56,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-09T20:41:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-09T20:41:56,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-09T20:41:56,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 647 msec 2024-12-09T20:41:56,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 658 msec 2024-12-09T20:41:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:56,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T20:41:56,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:56,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:56,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:56,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:56,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:56,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:56,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/0388212f6ad8495db034a5c8adc50f06 is 50, key is test_row_0/A:col10/1733776916585/Put/seqid=0 2024-12-09T20:41:56,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741915_1091 (size=12301) 2024-12-09T20:41:56,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/0388212f6ad8495db034a5c8adc50f06 2024-12-09T20:41:56,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/81f9d54a3fda4e078d4b11dbdc3075d9 is 50, key is test_row_0/B:col10/1733776916585/Put/seqid=0 2024-12-09T20:41:56,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741916_1092 (size=12301) 2024-12-09T20:41:56,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/81f9d54a3fda4e078d4b11dbdc3075d9 2024-12-09T20:41:56,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/d93353e97b004d84a6a4a0d7bc10c68a is 50, key is test_row_0/C:col10/1733776916585/Put/seqid=0 2024-12-09T20:41:56,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776976698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:56,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741917_1093 (size=12301) 2024-12-09T20:41:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776976812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T20:41:56,931 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:56,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-09T20:41:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T20:41:56,934 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:56,934 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:56,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:57,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776977021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T20:41:57,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T20:41:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:57,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/d93353e97b004d84a6a4a0d7bc10c68a 2024-12-09T20:41:57,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/0388212f6ad8495db034a5c8adc50f06 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/0388212f6ad8495db034a5c8adc50f06 2024-12-09T20:41:57,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/0388212f6ad8495db034a5c8adc50f06, entries=150, sequenceid=394, filesize=12.0 K 2024-12-09T20:41:57,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/81f9d54a3fda4e078d4b11dbdc3075d9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/81f9d54a3fda4e078d4b11dbdc3075d9 2024-12-09T20:41:57,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/81f9d54a3fda4e078d4b11dbdc3075d9, entries=150, sequenceid=394, filesize=12.0 K 2024-12-09T20:41:57,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/d93353e97b004d84a6a4a0d7bc10c68a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d93353e97b004d84a6a4a0d7bc10c68a 2024-12-09T20:41:57,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d93353e97b004d84a6a4a0d7bc10c68a, entries=150, sequenceid=394, filesize=12.0 K 2024-12-09T20:41:57,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for dd7288626d8fbb1461f9ccf6551f1bf6 in 534ms, sequenceid=394, compaction requested=true 2024-12-09T20:41:57,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:57,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:57,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:57,130 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:41:57,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:57,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:57,130 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:41:57,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:57,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:57,132 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:41:57,132 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:41:57,132 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,132 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b48f1bb7667e4b9f953b73955d804a05, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0364fd5255384d49a93d7e53f53c8c59, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/bd2e936328ac4f95a59d53e83b1b724f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/81f9d54a3fda4e078d4b11dbdc3075d9] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=48.8 K 2024-12-09T20:41:57,132 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:41:57,132 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:41:57,132 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,132 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e8c1ea7b957c49ad80d56a937bff1d53, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/3255840b9f70453f9cd1d97a78bec6e9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c718c9f01f8e4c9f8acfc668963c1755, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/0388212f6ad8495db034a5c8adc50f06] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=48.8 K 2024-12-09T20:41:57,133 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b48f1bb7667e4b9f953b73955d804a05, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733776913171 2024-12-09T20:41:57,133 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c1ea7b957c49ad80d56a937bff1d53, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733776913171 2024-12-09T20:41:57,133 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0364fd5255384d49a93d7e53f53c8c59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733776914368 2024-12-09T20:41:57,133 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3255840b9f70453f9cd1d97a78bec6e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733776914368 2024-12-09T20:41:57,133 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2e936328ac4f95a59d53e83b1b724f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733776915057 2024-12-09T20:41:57,133 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting c718c9f01f8e4c9f8acfc668963c1755, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733776915057 2024-12-09T20:41:57,134 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0388212f6ad8495db034a5c8adc50f06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733776916232 2024-12-09T20:41:57,134 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f9d54a3fda4e078d4b11dbdc3075d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733776916232 2024-12-09T20:41:57,144 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:57,144 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:57,144 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/6e1d5998f20c4b6b97830074a5db7106 is 50, key is test_row_0/B:col10/1733776916585/Put/seqid=0 2024-12-09T20:41:57,145 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/c0c2d3b40d744650b0c69b9e68a390a6 is 50, key is test_row_0/A:col10/1733776916585/Put/seqid=0 2024-12-09T20:41:57,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741918_1094 (size=13221) 2024-12-09T20:41:57,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741919_1095 (size=13221) 2024-12-09T20:41:57,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T20:41:57,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,241 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T20:41:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/646cedbfcd46440fa7f03d5ab3a3bfe6 is 50, key is test_row_0/A:col10/1733776916690/Put/seqid=0 2024-12-09T20:41:57,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741920_1096 (size=12301) 2024-12-09T20:41:57,251 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/646cedbfcd46440fa7f03d5ab3a3bfe6 2024-12-09T20:41:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T20:41:57,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/c182c08955394cc79b659eab2216c7d4 is 50, key is test_row_0/B:col10/1733776916690/Put/seqid=0 2024-12-09T20:41:57,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741921_1097 (size=12301) 2024-12-09T20:41:57,265 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/c182c08955394cc79b659eab2216c7d4 2024-12-09T20:41:57,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/3e14f68ef0374d888958ff1f46b3c10b is 50, key is test_row_0/C:col10/1733776916690/Put/seqid=0 2024-12-09T20:41:57,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741922_1098 (size=12301) 2024-12-09T20:41:57,284 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/3e14f68ef0374d888958ff1f46b3c10b 2024-12-09T20:41:57,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/646cedbfcd46440fa7f03d5ab3a3bfe6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/646cedbfcd46440fa7f03d5ab3a3bfe6 2024-12-09T20:41:57,296 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/646cedbfcd46440fa7f03d5ab3a3bfe6, entries=150, sequenceid=412, filesize=12.0 K 2024-12-09T20:41:57,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/c182c08955394cc79b659eab2216c7d4 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c182c08955394cc79b659eab2216c7d4 2024-12-09T20:41:57,305 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c182c08955394cc79b659eab2216c7d4, entries=150, sequenceid=412, filesize=12.0 K 2024-12-09T20:41:57,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/3e14f68ef0374d888958ff1f46b3c10b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3e14f68ef0374d888958ff1f46b3c10b 2024-12-09T20:41:57,314 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3e14f68ef0374d888958ff1f46b3c10b, entries=150, sequenceid=412, filesize=12.0 K 2024-12-09T20:41:57,315 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for dd7288626d8fbb1461f9ccf6551f1bf6 in 74ms, sequenceid=412, compaction requested=true 2024-12-09T20:41:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-09T20:41:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-09T20:41:57,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T20:41:57,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 383 msec 2024-12-09T20:41:57,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 390 msec 2024-12-09T20:41:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:57,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:41:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:57,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/35dabbd534bf47f595fa6e6f8aca7da9 is 50, key is test_row_0/A:col10/1733776917346/Put/seqid=0 2024-12-09T20:41:57,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741923_1099 (size=14741) 2024-12-09T20:41:57,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776917400, max timestamp=1733776917414Num uniques 859; 2024-12-09T20:41:57,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:41:57,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776917400, max timestamp=1733776917414Num uniques 859; 2024-12-09T20:41:57,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776917400, max timestamp=1733776917414Num uniques 859; 2024-12-09T20:41:57,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:41:57,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:41:57,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776977478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:57,555 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/c0c2d3b40d744650b0c69b9e68a390a6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c0c2d3b40d744650b0c69b9e68a390a6 2024-12-09T20:41:57,556 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/6e1d5998f20c4b6b97830074a5db7106 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6e1d5998f20c4b6b97830074a5db7106 2024-12-09T20:41:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T20:41:57,561 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:41:57,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:41:57,563 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into c0c2d3b40d744650b0c69b9e68a390a6(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:57,563 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:57,563 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=12, startTime=1733776917130; duration=0sec 2024-12-09T20:41:57,564 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:57,564 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:57,564 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T20:41:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-09T20:41:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:41:57,566 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:41:57,568 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into 6e1d5998f20c4b6b97830074a5db7106(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:57,568 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:57,568 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=12, startTime=1733776917130; duration=0sec 2024-12-09T20:41:57,568 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:57,568 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:57,568 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:41:57,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:41:57,568 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T20:41:57,569 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:41:57,569 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,569 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6d829b2770654964a27f54083c26ddf2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2409f680da3b4f679903f2552f492e3f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/a51edf690b2a427da34f121cf8ba6bdf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d93353e97b004d84a6a4a0d7bc10c68a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3e14f68ef0374d888958ff1f46b3c10b] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=60.8 K 2024-12-09T20:41:57,569 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d829b2770654964a27f54083c26ddf2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733776913171 2024-12-09T20:41:57,570 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2409f680da3b4f679903f2552f492e3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733776914368 2024-12-09T20:41:57,570 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting a51edf690b2a427da34f121cf8ba6bdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733776915057 2024-12-09T20:41:57,570 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d93353e97b004d84a6a4a0d7bc10c68a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733776916232 2024-12-09T20:41:57,571 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e14f68ef0374d888958ff1f46b3c10b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733776916672 2024-12-09T20:41:57,590 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:57,590 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/305193d890964257bf0f2d3558a128c5 is 50, key is test_row_0/C:col10/1733776916690/Put/seqid=0 2024-12-09T20:41:57,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776977591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:57,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741924_1100 (size=13255) 2024-12-09T20:41:57,607 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/305193d890964257bf0f2d3558a128c5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/305193d890964257bf0f2d3558a128c5 2024-12-09T20:41:57,614 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into 305193d890964257bf0f2d3558a128c5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:57,615 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:57,615 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=11, startTime=1733776917130; duration=0sec 2024-12-09T20:41:57,615 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:57,615 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:41:57,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:57,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/35dabbd534bf47f595fa6e6f8aca7da9 2024-12-09T20:41:57,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/33749f783d994cb38ae3f43fe96943f3 is 50, key is test_row_0/B:col10/1733776917346/Put/seqid=0 2024-12-09T20:41:57,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741925_1101 (size=12301) 2024-12-09T20:41:57,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776977801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:57,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:57,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:57,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:57,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:57,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:57,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:57,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:57,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:41:58,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:58,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776978110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:58,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/33749f783d994cb38ae3f43fe96943f3 2024-12-09T20:41:58,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6572a26db73e4cd6a449e396f2f3ecd6 is 50, key is test_row_0/C:col10/1733776917346/Put/seqid=0 2024-12-09T20:41:58,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:58,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741926_1102 (size=12301) 2024-12-09T20:41:58,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:58,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:41:58,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:58,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:58,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:58,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:58,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6572a26db73e4cd6a449e396f2f3ecd6 2024-12-09T20:41:58,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/35dabbd534bf47f595fa6e6f8aca7da9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35dabbd534bf47f595fa6e6f8aca7da9 2024-12-09T20:41:58,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35dabbd534bf47f595fa6e6f8aca7da9, entries=200, sequenceid=422, filesize=14.4 K 2024-12-09T20:41:58,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/33749f783d994cb38ae3f43fe96943f3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/33749f783d994cb38ae3f43fe96943f3 2024-12-09T20:41:58,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/33749f783d994cb38ae3f43fe96943f3, entries=150, sequenceid=422, filesize=12.0 K 2024-12-09T20:41:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/6572a26db73e4cd6a449e396f2f3ecd6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6572a26db73e4cd6a449e396f2f3ecd6 2024-12-09T20:41:58,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6572a26db73e4cd6a449e396f2f3ecd6, entries=150, sequenceid=422, filesize=12.0 K 2024-12-09T20:41:58,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for dd7288626d8fbb1461f9ccf6551f1bf6 in 1258ms, sequenceid=422, compaction requested=true 2024-12-09T20:41:58,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:58,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:58,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:58,604 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:58,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:58,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:58,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:58,605 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:58,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:58,605 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:58,605 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:58,606 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:41:58,606 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:41:58,606 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,606 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,606 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c0c2d3b40d744650b0c69b9e68a390a6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/646cedbfcd46440fa7f03d5ab3a3bfe6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35dabbd534bf47f595fa6e6f8aca7da9] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=39.3 K 2024-12-09T20:41:58,606 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6e1d5998f20c4b6b97830074a5db7106, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c182c08955394cc79b659eab2216c7d4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/33749f783d994cb38ae3f43fe96943f3] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=36.9 K 2024-12-09T20:41:58,606 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0c2d3b40d744650b0c69b9e68a390a6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733776916232 2024-12-09T20:41:58,606 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e1d5998f20c4b6b97830074a5db7106, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733776916232 2024-12-09T20:41:58,607 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 646cedbfcd46440fa7f03d5ab3a3bfe6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733776916672 2024-12-09T20:41:58,607 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c182c08955394cc79b659eab2216c7d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733776916672 2024-12-09T20:41:58,607 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35dabbd534bf47f595fa6e6f8aca7da9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733776917331 2024-12-09T20:41:58,607 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 33749f783d994cb38ae3f43fe96943f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733776917331 2024-12-09T20:41:58,623 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:58,623 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/e12a58bad18d427dbbedfc8ca0a797f0 is 50, key is test_row_0/A:col10/1733776917346/Put/seqid=0 2024-12-09T20:41:58,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733776917464, max timestamp=1733776918621Num uniques 1018; 2024-12-09T20:41:58,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:41:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:58,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=102.87 KB heapSize=292.20 KB 2024-12-09T20:41:58,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:58,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:41:58,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:58,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:41:58,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:58,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:41:58,634 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:58,635 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/a22f8af55dc94a36a129e8a4928f3f88 is 50, key is test_row_0/B:col10/1733776917346/Put/seqid=0 2024-12-09T20:41:58,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/7a500eda91574c7cb34eb6e235f947fb is 50, key is test_row_0/A:col10/1733776918621/Put/seqid=0 2024-12-09T20:41:58,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:58,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:58,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741927_1103 (size=13323) 2024-12-09T20:41:58,657 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/e12a58bad18d427dbbedfc8ca0a797f0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e12a58bad18d427dbbedfc8ca0a797f0 2024-12-09T20:41:58,666 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into e12a58bad18d427dbbedfc8ca0a797f0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:58,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:58,666 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=13, startTime=1733776918604; duration=0sec 2024-12-09T20:41:58,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:58,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:58,667 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T20:41:58,667 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T20:41:58,668 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T20:41:58,668 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. because compaction request was cancelled 2024-12-09T20:41:58,668 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:58,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741928_1104 (size=14741) 2024-12-09T20:41:58,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/7a500eda91574c7cb34eb6e235f947fb 2024-12-09T20:41:58,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741929_1105 (size=13323) 2024-12-09T20:41:58,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/d7116d722b5141fd99213a28495a81fb is 50, key is test_row_0/B:col10/1733776918621/Put/seqid=0 2024-12-09T20:41:58,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776978688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:58,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741930_1106 (size=12301) 2024-12-09T20:41:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:41:58,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:58,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:58,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776978801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:58,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:58,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:58,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:58,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776979011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:59,087 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/a22f8af55dc94a36a129e8a4928f3f88 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/a22f8af55dc94a36a129e8a4928f3f88 2024-12-09T20:41:59,092 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into a22f8af55dc94a36a129e8a4928f3f88(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:59,092 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:59,092 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=13, startTime=1733776918604; duration=0sec 2024-12-09T20:41:59,092 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:59,092 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:59,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/d7116d722b5141fd99213a28495a81fb 2024-12-09T20:41:59,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:59,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:59,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:59,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:59,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/de77308311ad44e895e8ebbeedfe01b7 is 50, key is test_row_0/C:col10/1733776918621/Put/seqid=0 2024-12-09T20:41:59,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741931_1107 (size=12301) 2024-12-09T20:41:59,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:59,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776979321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:59,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:59,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:41:59,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/de77308311ad44e895e8ebbeedfe01b7 2024-12-09T20:41:59,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/7a500eda91574c7cb34eb6e235f947fb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/7a500eda91574c7cb34eb6e235f947fb 2024-12-09T20:41:59,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/7a500eda91574c7cb34eb6e235f947fb, entries=200, sequenceid=463, filesize=14.4 K 2024-12-09T20:41:59,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/d7116d722b5141fd99213a28495a81fb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d7116d722b5141fd99213a28495a81fb 2024-12-09T20:41:59,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d7116d722b5141fd99213a28495a81fb, entries=150, sequenceid=463, filesize=12.0 K 2024-12-09T20:41:59,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/de77308311ad44e895e8ebbeedfe01b7 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/de77308311ad44e895e8ebbeedfe01b7 2024-12-09T20:41:59,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/de77308311ad44e895e8ebbeedfe01b7, entries=150, sequenceid=463, filesize=12.0 K 2024-12-09T20:41:59,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~292.16 KB/299168, currentSize=73.80 KB/75570 for dd7288626d8fbb1461f9ccf6551f1bf6 in 908ms, sequenceid=463, compaction requested=true 2024-12-09T20:41:59,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:59,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T20:41:59,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:59,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:41:59,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T20:41:59,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:41:59,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. because compaction request was cancelled 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. because compaction request was cancelled 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:41:59,537 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:41:59,538 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:41:59,539 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:41:59,539 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:41:59,539 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:59,539 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/305193d890964257bf0f2d3558a128c5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6572a26db73e4cd6a449e396f2f3ecd6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/de77308311ad44e895e8ebbeedfe01b7] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=37.0 K 2024-12-09T20:41:59,539 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 305193d890964257bf0f2d3558a128c5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733776916672 2024-12-09T20:41:59,540 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6572a26db73e4cd6a449e396f2f3ecd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733776917331 2024-12-09T20:41:59,540 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting de77308311ad44e895e8ebbeedfe01b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733776917464 2024-12-09T20:41:59,547 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:41:59,548 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2b2199d5afc54d9dad799f18573ff633 is 50, key is test_row_0/C:col10/1733776918621/Put/seqid=0 2024-12-09T20:41:59,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741932_1108 (size=13357) 2024-12-09T20:41:59,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T20:41:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:41:59,560 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T20:41:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:41:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:41:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:41:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:41:59,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/6ef4bc26a7d145d2b85bbfeaec25f0f1 is 50, key is test_row_0/A:col10/1733776918683/Put/seqid=0 2024-12-09T20:41:59,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741933_1109 (size=12301) 2024-12-09T20:41:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:41:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:41:59,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:41:59,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776979867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:59,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1733776979871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:59,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60306 deadline: 1733776979871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:59,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60358 deadline: 1733776979871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:59,872 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:59,873 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,873 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60340 deadline: 1733776979901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:59,905 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in dd7288626d8fbb1461f9ccf6551f1bf6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:41:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:59,958 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/2b2199d5afc54d9dad799f18573ff633 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2b2199d5afc54d9dad799f18573ff633 2024-12-09T20:41:59,963 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into 2b2199d5afc54d9dad799f18573ff633(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:41:59,963 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:41:59,963 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=13, startTime=1733776919537; duration=0sec 2024-12-09T20:41:59,964 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:41:59,964 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:41:59,971 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/6ef4bc26a7d145d2b85bbfeaec25f0f1 2024-12-09T20:41:59,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:41:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776979971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:41:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:41:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:41:59,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/c4f312674cd84d7d94fcf43e5da69154 is 50, key is test_row_0/B:col10/1733776918683/Put/seqid=0 2024-12-09T20:41:59,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741934_1110 (size=12301) 2024-12-09T20:42:00,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776980181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:00,383 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/c4f312674cd84d7d94fcf43e5da69154 2024-12-09T20:42:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/0e4490aa1b164aebae6e82cbb91a5f3c is 50, key is test_row_0/C:col10/1733776918683/Put/seqid=0 2024-12-09T20:42:00,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741935_1111 (size=12301) 2024-12-09T20:42:00,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776980491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:00,807 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/0e4490aa1b164aebae6e82cbb91a5f3c 2024-12-09T20:42:00,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/6ef4bc26a7d145d2b85bbfeaec25f0f1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/6ef4bc26a7d145d2b85bbfeaec25f0f1 2024-12-09T20:42:00,821 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/6ef4bc26a7d145d2b85bbfeaec25f0f1, entries=150, sequenceid=479, filesize=12.0 K 2024-12-09T20:42:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/c4f312674cd84d7d94fcf43e5da69154 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c4f312674cd84d7d94fcf43e5da69154 2024-12-09T20:42:00,828 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c4f312674cd84d7d94fcf43e5da69154, entries=150, sequenceid=479, filesize=12.0 K 2024-12-09T20:42:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/0e4490aa1b164aebae6e82cbb91a5f3c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0e4490aa1b164aebae6e82cbb91a5f3c 2024-12-09T20:42:00,837 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0e4490aa1b164aebae6e82cbb91a5f3c, entries=150, sequenceid=479, filesize=12.0 K 2024-12-09T20:42:00,838 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for dd7288626d8fbb1461f9ccf6551f1bf6 in 1277ms, sequenceid=479, compaction requested=true 2024-12-09T20:42:00,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:42:00,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:00,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-09T20:42:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-09T20:42:00,841 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T20:42:00,841 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2710 sec 2024-12-09T20:42:00,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 3.2800 sec 2024-12-09T20:42:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:42:01,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:42:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:42:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:42:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:42:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:01,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/35691f6a539c423395817c5a3bb0343c is 50, key is test_row_0/A:col10/1733776919852/Put/seqid=0 2024-12-09T20:42:01,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741936_1112 (size=14741) 2024-12-09T20:42:01,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776981035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:01,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776981141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:01,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776981351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:01,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/35691f6a539c423395817c5a3bb0343c 2024-12-09T20:42:01,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0e739ed920534a328cae228c65aabf9b is 50, key is test_row_0/B:col10/1733776919852/Put/seqid=0 2024-12-09T20:42:01,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741937_1113 (size=12301) 2024-12-09T20:42:01,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776981661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:01,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:01,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:01,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:42:01,721 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:01,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-09T20:42:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T20:42:01,723 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:01,724 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:01,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:01,726 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-09T20:42:01,727 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:01,727 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-09T20:42:01,727 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:01,728 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-09T20:42:01,728 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:01,728 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-09T20:42:01,728 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:01,731 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:01,731 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:01,731 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:01,731 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:01,731 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:01,731 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:01,731 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:01,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:01,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:01,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:01,732 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:01,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T20:42:01,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0e739ed920534a328cae228c65aabf9b 2024-12-09T20:42:01,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/81c8f39b4b904e258f02e108245fdd24 is 50, key is test_row_0/C:col10/1733776919852/Put/seqid=0 2024-12-09T20:42:01,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741938_1114 (size=12301) 2024-12-09T20:42:01,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-09T20:42:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:42:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:01,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:01,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:02,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-09T20:42:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:42:02,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:02,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:02,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T20:42:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60300 deadline: 1733776982172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:02,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-09T20:42:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:42:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:02,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:02,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/81c8f39b4b904e258f02e108245fdd24 2024-12-09T20:42:02,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/35691f6a539c423395817c5a3bb0343c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35691f6a539c423395817c5a3bb0343c 2024-12-09T20:42:02,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35691f6a539c423395817c5a3bb0343c, entries=200, sequenceid=499, filesize=14.4 K 2024-12-09T20:42:02,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/0e739ed920534a328cae228c65aabf9b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0e739ed920534a328cae228c65aabf9b 2024-12-09T20:42:02,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0e739ed920534a328cae228c65aabf9b, entries=150, sequenceid=499, filesize=12.0 K 2024-12-09T20:42:02,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/81c8f39b4b904e258f02e108245fdd24 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/81c8f39b4b904e258f02e108245fdd24 2024-12-09T20:42:02,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/81c8f39b4b904e258f02e108245fdd24, entries=150, sequenceid=499, filesize=12.0 K 2024-12-09T20:42:02,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for dd7288626d8fbb1461f9ccf6551f1bf6 in 1282ms, sequenceid=499, compaction requested=true 2024-12-09T20:42:02,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:42:02,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:02,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:02,285 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:02,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:02,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:02,285 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:02,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dd7288626d8fbb1461f9ccf6551f1bf6:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:02,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:02,287 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55106 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:02,287 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:02,287 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/B is initiating minor compaction (all files) 2024-12-09T20:42:02,287 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/A is initiating minor compaction (all files) 2024-12-09T20:42:02,287 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/A in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:02,287 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/B in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:02,287 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/a22f8af55dc94a36a129e8a4928f3f88, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d7116d722b5141fd99213a28495a81fb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c4f312674cd84d7d94fcf43e5da69154, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0e739ed920534a328cae228c65aabf9b] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=49.0 K 2024-12-09T20:42:02,287 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e12a58bad18d427dbbedfc8ca0a797f0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/7a500eda91574c7cb34eb6e235f947fb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/6ef4bc26a7d145d2b85bbfeaec25f0f1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35691f6a539c423395817c5a3bb0343c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=53.8 K 2024-12-09T20:42:02,287 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting a22f8af55dc94a36a129e8a4928f3f88, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733776917331 2024-12-09T20:42:02,287 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e12a58bad18d427dbbedfc8ca0a797f0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733776917331 2024-12-09T20:42:02,287 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d7116d722b5141fd99213a28495a81fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733776917464 2024-12-09T20:42:02,287 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a500eda91574c7cb34eb6e235f947fb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733776917464 2024-12-09T20:42:02,288 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ef4bc26a7d145d2b85bbfeaec25f0f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733776918633 2024-12-09T20:42:02,288 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f312674cd84d7d94fcf43e5da69154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733776918633 2024-12-09T20:42:02,288 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35691f6a539c423395817c5a3bb0343c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1733776919852 2024-12-09T20:42:02,288 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e739ed920534a328cae228c65aabf9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1733776919852 2024-12-09T20:42:02,296 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:02,296 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:02,296 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/233ff31904164c68bd02a49de9e89b4e is 50, key is test_row_0/B:col10/1733776919852/Put/seqid=0 2024-12-09T20:42:02,296 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/7e43464bb50d43ab91dd44c040e894e9 is 50, key is test_row_0/A:col10/1733776919852/Put/seqid=0 2024-12-09T20:42:02,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741939_1115 (size=13459) 2024-12-09T20:42:02,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741940_1116 (size=13459) 2024-12-09T20:42:02,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-09T20:42:02,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:02,345 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:42:02,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:42:02,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:02,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:42:02,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:02,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:42:02,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/fffc069e199444c08385435b605e544e is 50, key is test_row_0/A:col10/1733776921034/Put/seqid=0 2024-12-09T20:42:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T20:42:02,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741941_1117 (size=12301) 2024-12-09T20:42:02,713 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/7e43464bb50d43ab91dd44c040e894e9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/7e43464bb50d43ab91dd44c040e894e9 2024-12-09T20:42:02,713 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/233ff31904164c68bd02a49de9e89b4e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/233ff31904164c68bd02a49de9e89b4e 2024-12-09T20:42:02,719 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/B of dd7288626d8fbb1461f9ccf6551f1bf6 into 233ff31904164c68bd02a49de9e89b4e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:02,719 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/A of dd7288626d8fbb1461f9ccf6551f1bf6 into 7e43464bb50d43ab91dd44c040e894e9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:02,719 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:42:02,719 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:42:02,719 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/B, priority=12, startTime=1733776922285; duration=0sec 2024-12-09T20:42:02,719 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/A, priority=12, startTime=1733776922285; duration=0sec 2024-12-09T20:42:02,719 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:02,719 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:B 2024-12-09T20:42:02,719 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:02,719 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:02,719 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:A 2024-12-09T20:42:02,720 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:02,720 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): dd7288626d8fbb1461f9ccf6551f1bf6/C is initiating minor compaction (all files) 2024-12-09T20:42:02,720 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dd7288626d8fbb1461f9ccf6551f1bf6/C in TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:02,720 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2b2199d5afc54d9dad799f18573ff633, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0e4490aa1b164aebae6e82cbb91a5f3c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/81c8f39b4b904e258f02e108245fdd24] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp, totalSize=37.1 K 2024-12-09T20:42:02,721 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2199d5afc54d9dad799f18573ff633, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733776917464 2024-12-09T20:42:02,721 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e4490aa1b164aebae6e82cbb91a5f3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733776918633 2024-12-09T20:42:02,722 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 81c8f39b4b904e258f02e108245fdd24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1733776919852 2024-12-09T20:42:02,729 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dd7288626d8fbb1461f9ccf6551f1bf6#C#compaction#105 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:02,729 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/b952ce279cc249b8ac05797a92c372bf is 50, key is test_row_0/C:col10/1733776919852/Put/seqid=0 2024-12-09T20:42:02,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741942_1118 (size=13459) 2024-12-09T20:42:02,737 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/b952ce279cc249b8ac05797a92c372bf as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/b952ce279cc249b8ac05797a92c372bf 2024-12-09T20:42:02,742 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dd7288626d8fbb1461f9ccf6551f1bf6/C of dd7288626d8fbb1461f9ccf6551f1bf6 into b952ce279cc249b8ac05797a92c372bf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:02,742 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:42:02,742 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., storeName=dd7288626d8fbb1461f9ccf6551f1bf6/C, priority=13, startTime=1733776922285; duration=0sec 2024-12-09T20:42:02,743 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:02,743 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dd7288626d8fbb1461f9ccf6551f1bf6:C 2024-12-09T20:42:02,754 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/fffc069e199444c08385435b605e544e 2024-12-09T20:42:02,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/8f629f162f9c426da947a23c2567b138 is 50, key is test_row_0/B:col10/1733776921034/Put/seqid=0 2024-12-09T20:42:02,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741943_1119 (size=12301) 2024-12-09T20:42:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T20:42:03,170 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/8f629f162f9c426da947a23c2567b138 2024-12-09T20:42:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:42:03,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. as already flushing 2024-12-09T20:42:03,183 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-09T20:42:03,183 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:03,183 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:03,183 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:03,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/deb433748ed4478caf09b32e0447840e is 50, key is test_row_0/C:col10/1733776921034/Put/seqid=0 2024-12-09T20:42:03,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:03,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741944_1120 (size=12301) 2024-12-09T20:42:03,589 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/deb433748ed4478caf09b32e0447840e 2024-12-09T20:42:03,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/fffc069e199444c08385435b605e544e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fffc069e199444c08385435b605e544e 2024-12-09T20:42:03,605 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fffc069e199444c08385435b605e544e, entries=150, sequenceid=512, filesize=12.0 K 2024-12-09T20:42:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/8f629f162f9c426da947a23c2567b138 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/8f629f162f9c426da947a23c2567b138 2024-12-09T20:42:03,611 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/8f629f162f9c426da947a23c2567b138, entries=150, sequenceid=512, filesize=12.0 K 2024-12-09T20:42:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/deb433748ed4478caf09b32e0447840e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/deb433748ed4478caf09b32e0447840e 2024-12-09T20:42:03,618 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/deb433748ed4478caf09b32e0447840e, entries=150, sequenceid=512, filesize=12.0 K 2024-12-09T20:42:03,619 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for dd7288626d8fbb1461f9ccf6551f1bf6 in 1274ms, sequenceid=512, compaction requested=false 2024-12-09T20:42:03,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for dd7288626d8fbb1461f9ccf6551f1bf6: 2024-12-09T20:42:03,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:03,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-09T20:42:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-09T20:42:03,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-09T20:42:03,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-12-09T20:42:03,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.9020 sec 2024-12-09T20:42:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T20:42:03,883 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:04,113 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T20:42:09,938 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-09T20:42:09,938 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-09T20:42:09,938 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:09,938 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:09,938 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:09,939 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:09,939 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:09,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:09,939 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:09,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:09,953 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-09T20:42:09,953 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:09,953 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:09,953 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:09,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:10,012 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-09T20:42:10,013 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:10,013 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:10,013 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:10,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:10,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T20:42:10,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-09T20:42:10,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-12-09T20:42:10,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-09T20:42:10,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-09T20:42:10,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 178 2024-12-09T20:42:10,013 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T20:42:10,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3996 2024-12-09T20:42:10,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4006 2024-12-09T20:42:10,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T20:42:10,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1759 2024-12-09T20:42:10,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5270 rows 2024-12-09T20:42:10,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1758 2024-12-09T20:42:10,014 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5269 rows 2024-12-09T20:42:10,014 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:42:10,014 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:42:10,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:10,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:10,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:10,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T20:42:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T20:42:10,026 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776930026"}]},"ts":"1733776930026"} 2024-12-09T20:42:10,028 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T20:42:10,029 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T20:42:10,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T20:42:10,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd7288626d8fbb1461f9ccf6551f1bf6, UNASSIGN}] 2024-12-09T20:42:10,035 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd7288626d8fbb1461f9ccf6551f1bf6, UNASSIGN 2024-12-09T20:42:10,037 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=dd7288626d8fbb1461f9ccf6551f1bf6, regionState=CLOSING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:10,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd7288626d8fbb1461f9ccf6551f1bf6, UNASSIGN because future has completed 2024-12-09T20:42:10,039 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T20:42:10,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:42:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T20:42:10,197 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:42:10,197 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T20:42:10,198 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing dd7288626d8fbb1461f9ccf6551f1bf6, disabling compactions & flushes 2024-12-09T20:42:10,198 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:10,198 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:10,198 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. after waiting 0 ms 2024-12-09T20:42:10,198 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:10,198 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing dd7288626d8fbb1461f9ccf6551f1bf6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T20:42:10,199 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=A 2024-12-09T20:42:10,199 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:10,199 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=B 2024-12-09T20:42:10,199 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:10,199 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dd7288626d8fbb1461f9ccf6551f1bf6, store=C 2024-12-09T20:42:10,199 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:10,205 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/4edc9ba849b0466c89f2c0550ea02726 is 50, key is test_row_0/A:col10/1733776929933/Put/seqid=0 2024-12-09T20:42:10,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741945_1121 (size=12301) 2024-12-09T20:42:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T20:42:10,611 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/4edc9ba849b0466c89f2c0550ea02726 2024-12-09T20:42:10,624 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/f6dcb1b7846b46c58cc49f604f01adb6 is 50, key is test_row_0/B:col10/1733776929933/Put/seqid=0 2024-12-09T20:42:10,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741946_1122 (size=12301) 2024-12-09T20:42:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T20:42:11,030 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/f6dcb1b7846b46c58cc49f604f01adb6 2024-12-09T20:42:11,044 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/d59238484e234a62b8999d8604f098e8 is 50, key is test_row_0/C:col10/1733776929933/Put/seqid=0 2024-12-09T20:42:11,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741947_1123 (size=12301) 2024-12-09T20:42:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T20:42:11,451 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/d59238484e234a62b8999d8604f098e8 2024-12-09T20:42:11,465 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/A/4edc9ba849b0466c89f2c0550ea02726 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/4edc9ba849b0466c89f2c0550ea02726 2024-12-09T20:42:11,471 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/4edc9ba849b0466c89f2c0550ea02726, entries=150, sequenceid=523, filesize=12.0 K 2024-12-09T20:42:11,472 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/B/f6dcb1b7846b46c58cc49f604f01adb6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f6dcb1b7846b46c58cc49f604f01adb6 2024-12-09T20:42:11,477 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f6dcb1b7846b46c58cc49f604f01adb6, entries=150, sequenceid=523, filesize=12.0 K 2024-12-09T20:42:11,479 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/.tmp/C/d59238484e234a62b8999d8604f098e8 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d59238484e234a62b8999d8604f098e8 2024-12-09T20:42:11,483 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d59238484e234a62b8999d8604f098e8, entries=150, sequenceid=523, filesize=12.0 K 2024-12-09T20:42:11,484 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for dd7288626d8fbb1461f9ccf6551f1bf6 in 1286ms, sequenceid=523, compaction requested=true 2024-12-09T20:42:11,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/272d1ac76686429fb70d01f03ab734da, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a369d3da936a40088f682eecbbea2b3f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fcaa307c704d487e8541fa90270c6655, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/ff0cae8179584951b35e5eb73bf86354, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/50456e1520654fcb9706eb3dbb733868, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/effb4933321c4108a14fb40fba074195, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/095424b9641342c0b43795b15ebdc0ed, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d68f1e8df5e5443f9bd05c12d292a226, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d7787e0f7c22419eba5a44760453aae3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/8e51b0f29b994dcf85b830ddf3558137, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/70789f81fd1a43cb82d953927a8306f9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fbf03fd4c8b74558bccd5fbc66eba4b7, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a8754e0dab6146299315b3a388898d51, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c78edf52077748c5a7e7b7c52af5f5c2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/010555646c0146f09ce60e71364a6262, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/21192a7361af434aa8429354f5607ff2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/10ba878fee7d423d91f38e13673f7ad6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/97a354e3cf6c45ee915765075ea08e7c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e29505a12a7a4b2ea2350da0f2628b68, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/b6717ace514e4578a935021b04d3a27f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/023157ca7d6e41e28b68e6a183182d83, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/04485857216246abb4de5b4403a57115, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/74a340fae40f4e189cfe77bf9618ff39, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e8c1ea7b957c49ad80d56a937bff1d53, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/3255840b9f70453f9cd1d97a78bec6e9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c718c9f01f8e4c9f8acfc668963c1755, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c0c2d3b40d744650b0c69b9e68a390a6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/0388212f6ad8495db034a5c8adc50f06, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/646cedbfcd46440fa7f03d5ab3a3bfe6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35dabbd534bf47f595fa6e6f8aca7da9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e12a58bad18d427dbbedfc8ca0a797f0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/7a500eda91574c7cb34eb6e235f947fb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/6ef4bc26a7d145d2b85bbfeaec25f0f1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35691f6a539c423395817c5a3bb0343c] to archive 2024-12-09T20:42:11,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:42:11,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/272d1ac76686429fb70d01f03ab734da to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/272d1ac76686429fb70d01f03ab734da 2024-12-09T20:42:11,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a369d3da936a40088f682eecbbea2b3f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a369d3da936a40088f682eecbbea2b3f 2024-12-09T20:42:11,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fcaa307c704d487e8541fa90270c6655 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fcaa307c704d487e8541fa90270c6655 2024-12-09T20:42:11,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/ff0cae8179584951b35e5eb73bf86354 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/ff0cae8179584951b35e5eb73bf86354 2024-12-09T20:42:11,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/50456e1520654fcb9706eb3dbb733868 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/50456e1520654fcb9706eb3dbb733868 2024-12-09T20:42:11,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/effb4933321c4108a14fb40fba074195 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/effb4933321c4108a14fb40fba074195 2024-12-09T20:42:11,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/095424b9641342c0b43795b15ebdc0ed to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/095424b9641342c0b43795b15ebdc0ed 2024-12-09T20:42:11,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d68f1e8df5e5443f9bd05c12d292a226 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d68f1e8df5e5443f9bd05c12d292a226 2024-12-09T20:42:11,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d7787e0f7c22419eba5a44760453aae3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/d7787e0f7c22419eba5a44760453aae3 2024-12-09T20:42:11,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/8e51b0f29b994dcf85b830ddf3558137 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/8e51b0f29b994dcf85b830ddf3558137 2024-12-09T20:42:11,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/70789f81fd1a43cb82d953927a8306f9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/70789f81fd1a43cb82d953927a8306f9 2024-12-09T20:42:11,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fbf03fd4c8b74558bccd5fbc66eba4b7 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fbf03fd4c8b74558bccd5fbc66eba4b7 2024-12-09T20:42:11,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a8754e0dab6146299315b3a388898d51 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/a8754e0dab6146299315b3a388898d51 2024-12-09T20:42:11,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c78edf52077748c5a7e7b7c52af5f5c2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c78edf52077748c5a7e7b7c52af5f5c2 2024-12-09T20:42:11,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/010555646c0146f09ce60e71364a6262 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/010555646c0146f09ce60e71364a6262 2024-12-09T20:42:11,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/21192a7361af434aa8429354f5607ff2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/21192a7361af434aa8429354f5607ff2 2024-12-09T20:42:11,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/10ba878fee7d423d91f38e13673f7ad6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/10ba878fee7d423d91f38e13673f7ad6 2024-12-09T20:42:11,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/97a354e3cf6c45ee915765075ea08e7c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/97a354e3cf6c45ee915765075ea08e7c 2024-12-09T20:42:11,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e29505a12a7a4b2ea2350da0f2628b68 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e29505a12a7a4b2ea2350da0f2628b68 2024-12-09T20:42:11,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/b6717ace514e4578a935021b04d3a27f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/b6717ace514e4578a935021b04d3a27f 2024-12-09T20:42:11,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/023157ca7d6e41e28b68e6a183182d83 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/023157ca7d6e41e28b68e6a183182d83 2024-12-09T20:42:11,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/04485857216246abb4de5b4403a57115 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/04485857216246abb4de5b4403a57115 2024-12-09T20:42:11,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/74a340fae40f4e189cfe77bf9618ff39 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/74a340fae40f4e189cfe77bf9618ff39 2024-12-09T20:42:11,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e8c1ea7b957c49ad80d56a937bff1d53 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e8c1ea7b957c49ad80d56a937bff1d53 2024-12-09T20:42:11,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/3255840b9f70453f9cd1d97a78bec6e9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/3255840b9f70453f9cd1d97a78bec6e9 2024-12-09T20:42:11,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c718c9f01f8e4c9f8acfc668963c1755 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c718c9f01f8e4c9f8acfc668963c1755 2024-12-09T20:42:11,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c0c2d3b40d744650b0c69b9e68a390a6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/c0c2d3b40d744650b0c69b9e68a390a6 2024-12-09T20:42:11,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/0388212f6ad8495db034a5c8adc50f06 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/0388212f6ad8495db034a5c8adc50f06 2024-12-09T20:42:11,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/646cedbfcd46440fa7f03d5ab3a3bfe6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/646cedbfcd46440fa7f03d5ab3a3bfe6 2024-12-09T20:42:11,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35dabbd534bf47f595fa6e6f8aca7da9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35dabbd534bf47f595fa6e6f8aca7da9 2024-12-09T20:42:11,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e12a58bad18d427dbbedfc8ca0a797f0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/e12a58bad18d427dbbedfc8ca0a797f0 2024-12-09T20:42:11,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/7a500eda91574c7cb34eb6e235f947fb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/7a500eda91574c7cb34eb6e235f947fb 2024-12-09T20:42:11,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/6ef4bc26a7d145d2b85bbfeaec25f0f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/6ef4bc26a7d145d2b85bbfeaec25f0f1 2024-12-09T20:42:11,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35691f6a539c423395817c5a3bb0343c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/35691f6a539c423395817c5a3bb0343c 2024-12-09T20:42:11,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/1300c7ec0c9248a59c642078fb25f5e2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0b3ac6a3e956441aa3d5936f480f7d6a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6c734905a4904bbaa7def8933defff93, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/349522686b0845adb4b906faba2c957e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/00b795e7165848a9bdc18292f6263009, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f7162a5ddad9457b9ca030c4de6eb8b3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/56c3e1c7aaa342e382cdc6bf9fae2422, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0c1c47db05964bd9b7b2aaf114f24d6f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/fda3af51aa554808bc02ae8e0ebf48e5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/06acf1ecdbcb4f42a834d3984dd8082a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4768915f1aab4a36928911f74111ccd8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/667320b0029146689d9e534794520ea3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d6d94f2a8cef4ae5b6eb27578b533dee, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b1f37ee4a162419ea7498258302ef459, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6b9d1b46bb2347b99dfee8b17a75d3ae, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b53bbc5f53a7428bb3ad2336f57dbcb5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/54411d0f008b40cda8d178f692e841a6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/9fd0d7d2ae6446e8a4a0d62bd9f4a132, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4037069c058645fb946a355d66548fdf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/76151ea923c642ce93dec8adb97b9cbb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d92f3e3430ea429390768fab3bff6af2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/ba901999a91e4146a5aa45e138175dbb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b48f1bb7667e4b9f953b73955d804a05, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/5e994942c78b46e2962039bbad2fb9e8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0364fd5255384d49a93d7e53f53c8c59, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/bd2e936328ac4f95a59d53e83b1b724f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6e1d5998f20c4b6b97830074a5db7106, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/81f9d54a3fda4e078d4b11dbdc3075d9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c182c08955394cc79b659eab2216c7d4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/a22f8af55dc94a36a129e8a4928f3f88, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/33749f783d994cb38ae3f43fe96943f3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d7116d722b5141fd99213a28495a81fb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c4f312674cd84d7d94fcf43e5da69154, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0e739ed920534a328cae228c65aabf9b] to archive 2024-12-09T20:42:11,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:42:11,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/1300c7ec0c9248a59c642078fb25f5e2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/1300c7ec0c9248a59c642078fb25f5e2 2024-12-09T20:42:11,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0b3ac6a3e956441aa3d5936f480f7d6a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0b3ac6a3e956441aa3d5936f480f7d6a 2024-12-09T20:42:11,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6c734905a4904bbaa7def8933defff93 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6c734905a4904bbaa7def8933defff93 2024-12-09T20:42:11,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/349522686b0845adb4b906faba2c957e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/349522686b0845adb4b906faba2c957e 2024-12-09T20:42:11,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/00b795e7165848a9bdc18292f6263009 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/00b795e7165848a9bdc18292f6263009 2024-12-09T20:42:11,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f7162a5ddad9457b9ca030c4de6eb8b3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f7162a5ddad9457b9ca030c4de6eb8b3 2024-12-09T20:42:11,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/56c3e1c7aaa342e382cdc6bf9fae2422 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/56c3e1c7aaa342e382cdc6bf9fae2422 2024-12-09T20:42:11,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0c1c47db05964bd9b7b2aaf114f24d6f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0c1c47db05964bd9b7b2aaf114f24d6f 2024-12-09T20:42:11,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/fda3af51aa554808bc02ae8e0ebf48e5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/fda3af51aa554808bc02ae8e0ebf48e5 2024-12-09T20:42:11,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/06acf1ecdbcb4f42a834d3984dd8082a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/06acf1ecdbcb4f42a834d3984dd8082a 2024-12-09T20:42:11,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4768915f1aab4a36928911f74111ccd8 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4768915f1aab4a36928911f74111ccd8 2024-12-09T20:42:11,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/667320b0029146689d9e534794520ea3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/667320b0029146689d9e534794520ea3 2024-12-09T20:42:11,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d6d94f2a8cef4ae5b6eb27578b533dee to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d6d94f2a8cef4ae5b6eb27578b533dee 2024-12-09T20:42:11,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b1f37ee4a162419ea7498258302ef459 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b1f37ee4a162419ea7498258302ef459 2024-12-09T20:42:11,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6b9d1b46bb2347b99dfee8b17a75d3ae to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6b9d1b46bb2347b99dfee8b17a75d3ae 2024-12-09T20:42:11,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b53bbc5f53a7428bb3ad2336f57dbcb5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b53bbc5f53a7428bb3ad2336f57dbcb5 2024-12-09T20:42:11,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/54411d0f008b40cda8d178f692e841a6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/54411d0f008b40cda8d178f692e841a6 2024-12-09T20:42:11,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/9fd0d7d2ae6446e8a4a0d62bd9f4a132 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/9fd0d7d2ae6446e8a4a0d62bd9f4a132 2024-12-09T20:42:11,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4037069c058645fb946a355d66548fdf to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/4037069c058645fb946a355d66548fdf 2024-12-09T20:42:11,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/76151ea923c642ce93dec8adb97b9cbb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/76151ea923c642ce93dec8adb97b9cbb 2024-12-09T20:42:11,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d92f3e3430ea429390768fab3bff6af2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d92f3e3430ea429390768fab3bff6af2 2024-12-09T20:42:11,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/ba901999a91e4146a5aa45e138175dbb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/ba901999a91e4146a5aa45e138175dbb 2024-12-09T20:42:11,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b48f1bb7667e4b9f953b73955d804a05 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/b48f1bb7667e4b9f953b73955d804a05 2024-12-09T20:42:11,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/5e994942c78b46e2962039bbad2fb9e8 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/5e994942c78b46e2962039bbad2fb9e8 2024-12-09T20:42:11,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0364fd5255384d49a93d7e53f53c8c59 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0364fd5255384d49a93d7e53f53c8c59 2024-12-09T20:42:11,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/bd2e936328ac4f95a59d53e83b1b724f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/bd2e936328ac4f95a59d53e83b1b724f 2024-12-09T20:42:11,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6e1d5998f20c4b6b97830074a5db7106 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/6e1d5998f20c4b6b97830074a5db7106 2024-12-09T20:42:11,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/81f9d54a3fda4e078d4b11dbdc3075d9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/81f9d54a3fda4e078d4b11dbdc3075d9 2024-12-09T20:42:11,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c182c08955394cc79b659eab2216c7d4 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c182c08955394cc79b659eab2216c7d4 2024-12-09T20:42:11,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/a22f8af55dc94a36a129e8a4928f3f88 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/a22f8af55dc94a36a129e8a4928f3f88 2024-12-09T20:42:11,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/33749f783d994cb38ae3f43fe96943f3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/33749f783d994cb38ae3f43fe96943f3 2024-12-09T20:42:11,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d7116d722b5141fd99213a28495a81fb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/d7116d722b5141fd99213a28495a81fb 2024-12-09T20:42:11,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c4f312674cd84d7d94fcf43e5da69154 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/c4f312674cd84d7d94fcf43e5da69154 2024-12-09T20:42:11,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0e739ed920534a328cae228c65aabf9b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/0e739ed920534a328cae228c65aabf9b 2024-12-09T20:42:11,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73236cbd0de6440081cc78a185862842, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2a09840e449b42ec8ffb7a490ce6b794, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/40010b2b3e8142dba3ce60b44e29ae0c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2378d1c58b0441b88922afae9ecf16dc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3d12d889cff240fcbd8b46d20fa5b61b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6e94d56fea134f23adda3bd0536a5348, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6769edb8fd0d4b149a45b89a77bf4f3f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/17171fa2fc6940b9a071547cebf1ef8a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/f112d4a782a143eab1b897a5ad3fde91, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/522972781da0406d92a7dd98b966ca1c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/831c6f93a5404488a6fc51565c9bf36c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/38d8ab9363c64e828e18ef8a0d17bb8c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/9f628ba6656942feb71bcec8c6f78469, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0f82018aa7da41d2b71171febc0adadc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6f6e560424a94cbab3e3903c42565186, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/b8f008c5bc2641158d29a43e9576df0a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/8c38fc7255774ae0be9ce8fb232a787e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/111caa6f822a47cebe3c6082a947cd97, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6c68a6bba0f04485b7915e39848669a2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73a12cc3e81c498dbebc9f5c24a35e68, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/83d0549651cd4e1ab1737d0712ced939, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/86cd147c27224d938b5f90b6aa3be395, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6d829b2770654964a27f54083c26ddf2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2e7817f247fc451086c13f0b87a6d26c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2409f680da3b4f679903f2552f492e3f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/a51edf690b2a427da34f121cf8ba6bdf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d93353e97b004d84a6a4a0d7bc10c68a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/305193d890964257bf0f2d3558a128c5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3e14f68ef0374d888958ff1f46b3c10b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6572a26db73e4cd6a449e396f2f3ecd6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2b2199d5afc54d9dad799f18573ff633, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/de77308311ad44e895e8ebbeedfe01b7, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0e4490aa1b164aebae6e82cbb91a5f3c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/81c8f39b4b904e258f02e108245fdd24] to archive 2024-12-09T20:42:11,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:42:11,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73236cbd0de6440081cc78a185862842 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73236cbd0de6440081cc78a185862842 2024-12-09T20:42:11,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2a09840e449b42ec8ffb7a490ce6b794 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2a09840e449b42ec8ffb7a490ce6b794 2024-12-09T20:42:11,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/40010b2b3e8142dba3ce60b44e29ae0c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/40010b2b3e8142dba3ce60b44e29ae0c 2024-12-09T20:42:11,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2378d1c58b0441b88922afae9ecf16dc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2378d1c58b0441b88922afae9ecf16dc 2024-12-09T20:42:11,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3d12d889cff240fcbd8b46d20fa5b61b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3d12d889cff240fcbd8b46d20fa5b61b 2024-12-09T20:42:11,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6e94d56fea134f23adda3bd0536a5348 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6e94d56fea134f23adda3bd0536a5348 2024-12-09T20:42:11,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6769edb8fd0d4b149a45b89a77bf4f3f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6769edb8fd0d4b149a45b89a77bf4f3f 2024-12-09T20:42:11,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/17171fa2fc6940b9a071547cebf1ef8a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/17171fa2fc6940b9a071547cebf1ef8a 2024-12-09T20:42:11,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/f112d4a782a143eab1b897a5ad3fde91 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/f112d4a782a143eab1b897a5ad3fde91 2024-12-09T20:42:11,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/522972781da0406d92a7dd98b966ca1c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/522972781da0406d92a7dd98b966ca1c 2024-12-09T20:42:11,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/831c6f93a5404488a6fc51565c9bf36c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/831c6f93a5404488a6fc51565c9bf36c 2024-12-09T20:42:11,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/38d8ab9363c64e828e18ef8a0d17bb8c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/38d8ab9363c64e828e18ef8a0d17bb8c 2024-12-09T20:42:11,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/9f628ba6656942feb71bcec8c6f78469 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/9f628ba6656942feb71bcec8c6f78469 2024-12-09T20:42:11,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0f82018aa7da41d2b71171febc0adadc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0f82018aa7da41d2b71171febc0adadc 2024-12-09T20:42:11,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6f6e560424a94cbab3e3903c42565186 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6f6e560424a94cbab3e3903c42565186 2024-12-09T20:42:11,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/b8f008c5bc2641158d29a43e9576df0a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/b8f008c5bc2641158d29a43e9576df0a 2024-12-09T20:42:11,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/8c38fc7255774ae0be9ce8fb232a787e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/8c38fc7255774ae0be9ce8fb232a787e 2024-12-09T20:42:11,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/111caa6f822a47cebe3c6082a947cd97 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/111caa6f822a47cebe3c6082a947cd97 2024-12-09T20:42:11,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6c68a6bba0f04485b7915e39848669a2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6c68a6bba0f04485b7915e39848669a2 2024-12-09T20:42:11,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73a12cc3e81c498dbebc9f5c24a35e68 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/73a12cc3e81c498dbebc9f5c24a35e68 2024-12-09T20:42:11,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/83d0549651cd4e1ab1737d0712ced939 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/83d0549651cd4e1ab1737d0712ced939 2024-12-09T20:42:11,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/86cd147c27224d938b5f90b6aa3be395 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/86cd147c27224d938b5f90b6aa3be395 2024-12-09T20:42:11,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6d829b2770654964a27f54083c26ddf2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6d829b2770654964a27f54083c26ddf2 2024-12-09T20:42:11,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2e7817f247fc451086c13f0b87a6d26c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2e7817f247fc451086c13f0b87a6d26c 2024-12-09T20:42:11,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2409f680da3b4f679903f2552f492e3f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2409f680da3b4f679903f2552f492e3f 2024-12-09T20:42:11,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/a51edf690b2a427da34f121cf8ba6bdf to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/a51edf690b2a427da34f121cf8ba6bdf 2024-12-09T20:42:11,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d93353e97b004d84a6a4a0d7bc10c68a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d93353e97b004d84a6a4a0d7bc10c68a 2024-12-09T20:42:11,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/305193d890964257bf0f2d3558a128c5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/305193d890964257bf0f2d3558a128c5 2024-12-09T20:42:11,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3e14f68ef0374d888958ff1f46b3c10b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/3e14f68ef0374d888958ff1f46b3c10b 2024-12-09T20:42:11,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6572a26db73e4cd6a449e396f2f3ecd6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/6572a26db73e4cd6a449e396f2f3ecd6 2024-12-09T20:42:11,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2b2199d5afc54d9dad799f18573ff633 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/2b2199d5afc54d9dad799f18573ff633 2024-12-09T20:42:11,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/de77308311ad44e895e8ebbeedfe01b7 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/de77308311ad44e895e8ebbeedfe01b7 2024-12-09T20:42:11,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0e4490aa1b164aebae6e82cbb91a5f3c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/0e4490aa1b164aebae6e82cbb91a5f3c 2024-12-09T20:42:11,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/81c8f39b4b904e258f02e108245fdd24 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/81c8f39b4b904e258f02e108245fdd24 2024-12-09T20:42:11,646 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/recovered.edits/526.seqid, newMaxSeqId=526, maxSeqId=1 2024-12-09T20:42:11,647 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. 2024-12-09T20:42:11,647 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for dd7288626d8fbb1461f9ccf6551f1bf6: Waiting for close lock at 1733776930198Running coprocessor pre-close hooks at 1733776930198Disabling compacts and flushes for region at 1733776930198Disabling writes for close at 1733776930198Obtaining lock to block concurrent updates at 1733776930198Preparing flush snapshotting stores in dd7288626d8fbb1461f9ccf6551f1bf6 at 1733776930198Finished memstore snapshotting TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733776930199 (+1 ms)Flushing stores of TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6. at 1733776930200 (+1 ms)Flushing dd7288626d8fbb1461f9ccf6551f1bf6/A: creating writer at 1733776930200Flushing dd7288626d8fbb1461f9ccf6551f1bf6/A: appending metadata at 1733776930204 (+4 ms)Flushing dd7288626d8fbb1461f9ccf6551f1bf6/A: closing flushed file at 1733776930204Flushing dd7288626d8fbb1461f9ccf6551f1bf6/B: creating writer at 1733776930620 (+416 ms)Flushing dd7288626d8fbb1461f9ccf6551f1bf6/B: appending metadata at 1733776930624 (+4 ms)Flushing dd7288626d8fbb1461f9ccf6551f1bf6/B: closing flushed file at 1733776930624Flushing dd7288626d8fbb1461f9ccf6551f1bf6/C: creating writer at 1733776931039 (+415 ms)Flushing dd7288626d8fbb1461f9ccf6551f1bf6/C: appending metadata at 1733776931043 (+4 ms)Flushing dd7288626d8fbb1461f9ccf6551f1bf6/C: closing flushed file at 1733776931043Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27ef4d89: reopening flushed file at 1733776931464 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68cb44a2: reopening flushed file at 1733776931471 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@439dde44: reopening flushed file at 1733776931478 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for dd7288626d8fbb1461f9ccf6551f1bf6 in 1286ms, sequenceid=523, compaction requested=true at 1733776931484 (+6 ms)Writing region close event to WAL at 1733776931642 (+158 ms)Running coprocessor post-close hooks at 1733776931647 (+5 ms)Closed at 1733776931647 2024-12-09T20:42:11,649 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:42:11,650 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=dd7288626d8fbb1461f9ccf6551f1bf6, regionState=CLOSED 2024-12-09T20:42:11,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:42:11,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T20:42:11,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure dd7288626d8fbb1461f9ccf6551f1bf6, server=00c23a7251f8,36519,1733776895755 in 1.6120 sec 2024-12-09T20:42:11,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-09T20:42:11,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dd7288626d8fbb1461f9ccf6551f1bf6, UNASSIGN in 1.6200 sec 2024-12-09T20:42:11,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-09T20:42:11,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6260 sec 2024-12-09T20:42:11,658 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776931658"}]},"ts":"1733776931658"} 2024-12-09T20:42:11,660 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T20:42:11,660 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T20:42:11,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6400 sec 2024-12-09T20:42:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T20:42:12,173 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:12,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T20:42:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:12,189 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T20:42:12,191 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:12,195 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:42:12,199 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/recovered.edits] 2024-12-09T20:42:12,202 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/4edc9ba849b0466c89f2c0550ea02726 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/4edc9ba849b0466c89f2c0550ea02726 2024-12-09T20:42:12,203 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/7e43464bb50d43ab91dd44c040e894e9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/7e43464bb50d43ab91dd44c040e894e9 2024-12-09T20:42:12,204 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fffc069e199444c08385435b605e544e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/A/fffc069e199444c08385435b605e544e 2024-12-09T20:42:12,207 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/233ff31904164c68bd02a49de9e89b4e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/233ff31904164c68bd02a49de9e89b4e 2024-12-09T20:42:12,208 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/8f629f162f9c426da947a23c2567b138 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/8f629f162f9c426da947a23c2567b138 2024-12-09T20:42:12,209 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f6dcb1b7846b46c58cc49f604f01adb6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/B/f6dcb1b7846b46c58cc49f604f01adb6 2024-12-09T20:42:12,212 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/b952ce279cc249b8ac05797a92c372bf to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/b952ce279cc249b8ac05797a92c372bf 2024-12-09T20:42:12,213 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d59238484e234a62b8999d8604f098e8 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/d59238484e234a62b8999d8604f098e8 2024-12-09T20:42:12,215 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/deb433748ed4478caf09b32e0447840e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/C/deb433748ed4478caf09b32e0447840e 2024-12-09T20:42:12,218 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/recovered.edits/526.seqid to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6/recovered.edits/526.seqid 2024-12-09T20:42:12,218 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/dd7288626d8fbb1461f9ccf6551f1bf6 2024-12-09T20:42:12,218 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T20:42:12,223 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:12,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-09T20:42:12,231 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T20:42:12,259 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T20:42:12,260 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:12,260 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T20:42:12,261 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733776932261"}]},"ts":"9223372036854775807"} 2024-12-09T20:42:12,265 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T20:42:12,265 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dd7288626d8fbb1461f9ccf6551f1bf6, NAME => 'TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T20:42:12,265 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T20:42:12,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733776932265"}]},"ts":"9223372036854775807"} 2024-12-09T20:42:12,269 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T20:42:12,269 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:12,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-12-09T20:42:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T20:42:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T20:42:12,302 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:12,316 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_421848342_22 at /127.0.0.1:48558 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;00c23a7251f8:36519-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=340 (was 268) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3066 (was 3717) 2024-12-09T20:42:12,325 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=340, ProcessCount=11, AvailableMemoryMB=3066 2024-12-09T20:42:12,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T20:42:12,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:42:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:12,329 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:42:12,329 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:12,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-12-09T20:42:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T20:42:12,330 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:42:12,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741948_1124 (size=1061) 2024-12-09T20:42:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T20:42:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T20:42:12,742 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948 2024-12-09T20:42:12,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741949_1125 (size=53) 2024-12-09T20:42:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T20:42:13,156 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:42:13,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c7804971d6cc1c0c364cf6c88d0c46fe, disabling compactions & flushes 2024-12-09T20:42:13,157 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:13,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:13,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. after waiting 0 ms 2024-12-09T20:42:13,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:13,157 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:13,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c7804971d6cc1c0c364cf6c88d0c46fe: Waiting for close lock at 1733776933157Disabling compacts and flushes for region at 1733776933157Disabling writes for close at 1733776933157Writing region close event to WAL at 1733776933157Closed at 1733776933157 2024-12-09T20:42:13,161 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:42:13,162 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733776933161"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776933161"}]},"ts":"1733776933161"} 2024-12-09T20:42:13,167 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:42:13,168 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:42:13,168 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776933168"}]},"ts":"1733776933168"} 2024-12-09T20:42:13,170 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T20:42:13,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, ASSIGN}] 2024-12-09T20:42:13,172 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, ASSIGN 2024-12-09T20:42:13,173 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, ASSIGN; state=OFFLINE, location=00c23a7251f8,36519,1733776895755; forceNewPlan=false, retain=false 2024-12-09T20:42:13,324 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c7804971d6cc1c0c364cf6c88d0c46fe, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:13,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, ASSIGN because future has completed 2024-12-09T20:42:13,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:42:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T20:42:13,491 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:13,492 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:42:13,492 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,493 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:42:13,493 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,493 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,496 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,498 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:42:13,499 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7804971d6cc1c0c364cf6c88d0c46fe columnFamilyName A 2024-12-09T20:42:13,500 DEBUG [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:13,500 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(327): Store=c7804971d6cc1c0c364cf6c88d0c46fe/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:42:13,501 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,502 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:42:13,502 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7804971d6cc1c0c364cf6c88d0c46fe columnFamilyName B 2024-12-09T20:42:13,502 DEBUG [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:13,502 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(327): Store=c7804971d6cc1c0c364cf6c88d0c46fe/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:42:13,503 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,504 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:42:13,504 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7804971d6cc1c0c364cf6c88d0c46fe columnFamilyName C 2024-12-09T20:42:13,504 DEBUG [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:13,504 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(327): Store=c7804971d6cc1c0c364cf6c88d0c46fe/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:42:13,504 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:13,505 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,505 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,506 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,507 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,507 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,507 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:42:13,509 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,511 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:42:13,511 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened c7804971d6cc1c0c364cf6c88d0c46fe; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72711139, jitterRate=0.08348040282726288}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:42:13,511 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:13,512 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for c7804971d6cc1c0c364cf6c88d0c46fe: Running coprocessor pre-open hook at 1733776933494Writing region info on filesystem at 1733776933494Initializing all the Stores at 1733776933495 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776933495Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776933496 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776933496Cleaning up temporary data from old regions at 1733776933507 (+11 ms)Running coprocessor post-open hooks at 1733776933511 (+4 ms)Region opened successfully at 1733776933512 (+1 ms) 2024-12-09T20:42:13,513 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., pid=36, masterSystemTime=1733776933484 2024-12-09T20:42:13,515 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:13,515 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:13,515 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c7804971d6cc1c0c364cf6c88d0c46fe, regionState=OPEN, openSeqNum=2, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:13,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:42:13,518 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=00c23a7251f8,36519,1733776895755, table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T20:42:13,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-09T20:42:13,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 in 187 msec 2024-12-09T20:42:13,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-09T20:42:13,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, ASSIGN in 349 msec 2024-12-09T20:42:13,522 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:42:13,522 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776933522"}]},"ts":"1733776933522"} 2024-12-09T20:42:13,524 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T20:42:13,525 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:42:13,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1980 sec 2024-12-09T20:42:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T20:42:14,493 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:14,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:14,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:14,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:14,500 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f320694, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:14,504 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b0ecd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:14,505 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:14,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:14,509 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T20:42:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c23a7251f8,42351,1733776895022 2024-12-09T20:42:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1389b68a 2024-12-09T20:42:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T20:42:14,511 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T20:42:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T20:42:14,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:42:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:14,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741950_1126 (size=1097) 2024-12-09T20:42:14,992 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T20:42:14,992 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T20:42:15,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T20:42:15,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, REOPEN/MOVE}] 2024-12-09T20:42:15,010 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, REOPEN/MOVE 2024-12-09T20:42:15,011 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c7804971d6cc1c0c364cf6c88d0c46fe, regionState=CLOSING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, REOPEN/MOVE because future has completed 2024-12-09T20:42:15,013 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T20:42:15,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:42:15,166 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T20:42:15,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing c7804971d6cc1c0c364cf6c88d0c46fe, disabling compactions & flushes 2024-12-09T20:42:15,166 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. after waiting 0 ms 2024-12-09T20:42:15,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,172 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T20:42:15,173 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,173 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for c7804971d6cc1c0c364cf6c88d0c46fe: Waiting for close lock at 1733776935166Running coprocessor pre-close hooks at 1733776935166Disabling compacts and flushes for region at 1733776935166Disabling writes for close at 1733776935166Writing region close event to WAL at 1733776935167 (+1 ms)Running coprocessor post-close hooks at 1733776935173 (+6 ms)Closed at 1733776935173 2024-12-09T20:42:15,174 WARN [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: c7804971d6cc1c0c364cf6c88d0c46fe to self. 2024-12-09T20:42:15,177 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,178 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c7804971d6cc1c0c364cf6c88d0c46fe, regionState=CLOSED 2024-12-09T20:42:15,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:42:15,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-09T20:42:15,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 in 169 msec 2024-12-09T20:42:15,186 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, REOPEN/MOVE; state=CLOSED, location=00c23a7251f8,36519,1733776895755; forceNewPlan=false, retain=true 2024-12-09T20:42:15,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c7804971d6cc1c0c364cf6c88d0c46fe, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, REOPEN/MOVE because future has completed 2024-12-09T20:42:15,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:42:15,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T20:42:15,497 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,498 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:42:15,498 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,499 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:42:15,499 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,499 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,505 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,507 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:42:15,513 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7804971d6cc1c0c364cf6c88d0c46fe columnFamilyName A 2024-12-09T20:42:15,515 DEBUG [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:15,516 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(327): Store=c7804971d6cc1c0c364cf6c88d0c46fe/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:42:15,516 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,517 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:42:15,517 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7804971d6cc1c0c364cf6c88d0c46fe columnFamilyName B 2024-12-09T20:42:15,517 DEBUG [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:15,517 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(327): Store=c7804971d6cc1c0c364cf6c88d0c46fe/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:42:15,518 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,518 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:42:15,518 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7804971d6cc1c0c364cf6c88d0c46fe columnFamilyName C 2024-12-09T20:42:15,519 DEBUG [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:15,519 INFO [StoreOpener-c7804971d6cc1c0c364cf6c88d0c46fe-1 {}] regionserver.HStore(327): Store=c7804971d6cc1c0c364cf6c88d0c46fe/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:42:15,519 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,519 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,520 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,521 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,522 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,522 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,523 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:42:15,524 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,525 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened c7804971d6cc1c0c364cf6c88d0c46fe; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61156987, jitterRate=-0.0886898785829544}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:42:15,525 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,526 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for c7804971d6cc1c0c364cf6c88d0c46fe: Running coprocessor pre-open hook at 1733776935499Writing region info on filesystem at 1733776935499Initializing all the Stores at 1733776935501 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776935501Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776935501Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776935501Cleaning up temporary data from old regions at 1733776935522 (+21 ms)Running coprocessor post-open hooks at 1733776935525 (+3 ms)Region opened successfully at 1733776935526 (+1 ms) 2024-12-09T20:42:15,527 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., pid=41, masterSystemTime=1733776935492 2024-12-09T20:42:15,529 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,529 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,530 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c7804971d6cc1c0c364cf6c88d0c46fe, regionState=OPEN, openSeqNum=5, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:42:15,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-12-09T20:42:15,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 in 193 msec 2024-12-09T20:42:15,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-09T20:42:15,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, REOPEN/MOVE in 525 msec 2024-12-09T20:42:15,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-09T20:42:15,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 537 msec 2024-12-09T20:42:15,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0190 sec 2024-12-09T20:42:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-09T20:42:15,543 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:15,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:15,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:15,544 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3194a514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:15,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4551d57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:15,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:15,547 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243f4e8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:15,549 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27854e4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:15,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:15,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f69bfac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:15,557 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f7f07a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:15,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:15,560 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10dda74e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:15,562 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70104fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:15,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:15,565 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26901167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:15,567 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38497cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:15,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:15,574 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631f3ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:15,576 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c64b4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:15,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:15,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69153b88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:15,584 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a063abb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:15,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:15,587 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@166ebb22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:15,589 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405caf2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:15,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:15,594 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b1e567, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:15,596 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bae89bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:15,598 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:15,598 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:15,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:15,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:42:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:42:15,610 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:15,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:42:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:15,614 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:15,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-09T20:42:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T20:42:15,617 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:15,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:15,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:42:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:42:15,622 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:15,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:15,634 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:15,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:42:15,650 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:15,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:42:15,661 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:15,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:42:15,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776995667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776995668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,676 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1733776995669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51998 deadline: 1733776995669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:15,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:15,682 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:15,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:15,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776935645, max timestamp=1733776935661Num uniques 859; 2024-12-09T20:42:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:42:15,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:15,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776935645, max timestamp=1733776935661Num uniques 859; 2024-12-09T20:42:15,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:15,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776935645, max timestamp=1733776935661Num uniques 859; 2024-12-09T20:42:15,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:42:15,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209bc94c8c7642e4c8e9371863977da0d4b_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776935606/Put/seqid=0 2024-12-09T20:42:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T20:42:15,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741951_1127 (size=9714) 2024-12-09T20:42:15,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:15,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209bc94c8c7642e4c8e9371863977da0d4b_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bc94c8c7642e4c8e9371863977da0d4b_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:15,758 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/d418365a90de4a94b840cc73c49ef42e, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:15,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/d418365a90de4a94b840cc73c49ef42e is 175, key is test_row_0/A:col10/1733776935606/Put/seqid=0 2024-12-09T20:42:15,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T20:42:15,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:15,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:15,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:15,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1733776995782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51998 deadline: 1733776995783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776995784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776995786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776995787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741952_1128 (size=22361) 2024-12-09T20:42:15,815 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/d418365a90de4a94b840cc73c49ef42e 2024-12-09T20:42:15,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/0453aabb3ab4466bafb6834575d65dd2 is 50, key is test_row_0/B:col10/1733776935606/Put/seqid=0 2024-12-09T20:42:15,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741953_1129 (size=9657) 2024-12-09T20:42:15,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/0453aabb3ab4466bafb6834575d65dd2 2024-12-09T20:42:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776995892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776995901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776995901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:15,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T20:42:15,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T20:42:15,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:15,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:15,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:15,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:15,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/687f51b6dd124ecbb2641730c8db765e is 50, key is test_row_0/C:col10/1733776935606/Put/seqid=0 2024-12-09T20:42:15,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741954_1130 (size=9657) 2024-12-09T20:42:15,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/687f51b6dd124ecbb2641730c8db765e 2024-12-09T20:42:15,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/d418365a90de4a94b840cc73c49ef42e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d418365a90de4a94b840cc73c49ef42e 2024-12-09T20:42:16,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1733776995992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51998 deadline: 1733776995993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d418365a90de4a94b840cc73c49ef42e, entries=100, sequenceid=14, filesize=21.8 K 2024-12-09T20:42:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/0453aabb3ab4466bafb6834575d65dd2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0453aabb3ab4466bafb6834575d65dd2 2024-12-09T20:42:16,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0453aabb3ab4466bafb6834575d65dd2, entries=100, sequenceid=14, filesize=9.4 K 2024-12-09T20:42:16,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/687f51b6dd124ecbb2641730c8db765e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/687f51b6dd124ecbb2641730c8db765e 2024-12-09T20:42:16,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/687f51b6dd124ecbb2641730c8db765e, entries=100, sequenceid=14, filesize=9.4 K 2024-12-09T20:42:16,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c7804971d6cc1c0c364cf6c88d0c46fe in 429ms, sequenceid=14, compaction requested=false 2024-12-09T20:42:16,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:16,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T20:42:16,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:16,090 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T20:42:16,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:16,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:16,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:16,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:16,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:16,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d61bb5a5ce814f07b028f9e8b1f822ee_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776935785/Put/seqid=0 2024-12-09T20:42:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741955_1131 (size=12154) 2024-12-09T20:42:16,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,171 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d61bb5a5ce814f07b028f9e8b1f822ee_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d61bb5a5ce814f07b028f9e8b1f822ee_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/fb95bc54a72c47f1b7770a26e3a6b798, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:16,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/fb95bc54a72c47f1b7770a26e3a6b798 is 175, key is test_row_0/A:col10/1733776935785/Put/seqid=0 2024-12-09T20:42:16,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741956_1132 (size=30955) 2024-12-09T20:42:16,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,186 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/fb95bc54a72c47f1b7770a26e3a6b798 2024-12-09T20:42:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776996167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776996167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776996173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/da57747d85df470e818c4777cd69ae35 is 50, key is test_row_0/B:col10/1733776935785/Put/seqid=0 2024-12-09T20:42:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T20:42:16,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741957_1133 (size=12001) 2024-12-09T20:42:16,250 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/da57747d85df470e818c4777cd69ae35 2024-12-09T20:42:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/e8b7ef1013f54289aeb86e8192a1b137 is 50, key is test_row_0/C:col10/1733776935785/Put/seqid=0 2024-12-09T20:42:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776996293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776996293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776996301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741958_1134 (size=12001) 2024-12-09T20:42:16,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51998 deadline: 1733776996312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1733776996312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,315 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/e8b7ef1013f54289aeb86e8192a1b137 2024-12-09T20:42:16,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/fb95bc54a72c47f1b7770a26e3a6b798 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/fb95bc54a72c47f1b7770a26e3a6b798 2024-12-09T20:42:16,331 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/fb95bc54a72c47f1b7770a26e3a6b798, entries=150, sequenceid=51, filesize=30.2 K 2024-12-09T20:42:16,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/da57747d85df470e818c4777cd69ae35 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/da57747d85df470e818c4777cd69ae35 2024-12-09T20:42:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,345 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/da57747d85df470e818c4777cd69ae35, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T20:42:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/e8b7ef1013f54289aeb86e8192a1b137 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/e8b7ef1013f54289aeb86e8192a1b137 2024-12-09T20:42:16,353 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/e8b7ef1013f54289aeb86e8192a1b137, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T20:42:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,354 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c7804971d6cc1c0c364cf6c88d0c46fe in 264ms, sequenceid=51, compaction requested=false 2024-12-09T20:42:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:16,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:16,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-09T20:42:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-09T20:42:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-09T20:42:16,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 734 msec 2024-12-09T20:42:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 747 msec 2024-12-09T20:42:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:16,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T20:42:16,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:16,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:16,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:16,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:16,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120915b8ccad69604b66b240a86406cdeead_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776936530/Put/seqid=0 2024-12-09T20:42:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741959_1135 (size=12154) 2024-12-09T20:42:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776996586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733776936560, max timestamp=1733776936587Num uniques 859; 2024-12-09T20:42:16,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733776936560, max timestamp=1733776936587Num uniques 859; 2024-12-09T20:42:16,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T20:42:16,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T20:42:16,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733776936560, max timestamp=1733776936587Num uniques 859; 2024-12-09T20:42:16,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T20:42:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776996634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776996646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776996701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T20:42:16,751 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:16,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-09T20:42:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T20:42:16,754 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:16,755 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:16,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776996751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776996771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1733776996821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51998 deadline: 1733776996821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T20:42:16,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T20:42:16,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:16,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:16,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:16,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:16,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:16,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776996912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776996961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,980 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:16,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776996981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:16,989 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120915b8ccad69604b66b240a86406cdeead_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120915b8ccad69604b66b240a86406cdeead_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:16,990 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/41a173f92ec741d29f83f50a586bda93, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:16,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/41a173f92ec741d29f83f50a586bda93 is 175, key is test_row_0/A:col10/1733776936530/Put/seqid=0 2024-12-09T20:42:17,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741960_1136 (size=30951) 2024-12-09T20:42:17,007 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/41a173f92ec741d29f83f50a586bda93 2024-12-09T20:42:17,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/ee7b00b3551146ef8d5916896a86f14a is 50, key is test_row_0/B:col10/1733776936530/Put/seqid=0 2024-12-09T20:42:17,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741961_1137 (size=9657) 2024-12-09T20:42:17,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T20:42:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T20:42:17,215 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:42:17,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T20:42:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776997223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:17,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776997271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776997291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:17,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T20:42:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T20:42:17,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/ee7b00b3551146ef8d5916896a86f14a 2024-12-09T20:42:17,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/f478e49650774032af2253d734de1203 is 50, key is test_row_0/C:col10/1733776936530/Put/seqid=0 2024-12-09T20:42:17,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741962_1138 (size=9657) 2024-12-09T20:42:17,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T20:42:17,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:17,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T20:42:17,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:17,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776997741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776997781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:17,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776997800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:17,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T20:42:17,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:17,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:17,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51998 deadline: 1733776997831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1733776997831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:17,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,834 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:17,834 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:17,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/f478e49650774032af2253d734de1203 2024-12-09T20:42:17,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/41a173f92ec741d29f83f50a586bda93 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/41a173f92ec741d29f83f50a586bda93 2024-12-09T20:42:17,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/41a173f92ec741d29f83f50a586bda93, entries=150, sequenceid=62, filesize=30.2 K 2024-12-09T20:42:17,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/ee7b00b3551146ef8d5916896a86f14a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ee7b00b3551146ef8d5916896a86f14a 2024-12-09T20:42:17,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ee7b00b3551146ef8d5916896a86f14a, entries=100, sequenceid=62, filesize=9.4 K 2024-12-09T20:42:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/f478e49650774032af2253d734de1203 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f478e49650774032af2253d734de1203 2024-12-09T20:42:17,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f478e49650774032af2253d734de1203, entries=100, sequenceid=62, filesize=9.4 K 2024-12-09T20:42:17,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c7804971d6cc1c0c364cf6c88d0c46fe in 1344ms, sequenceid=62, compaction requested=true 2024-12-09T20:42:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:17,877 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:17,877 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:17,878 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:17,878 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:17,878 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/B is initiating minor compaction (all files) 2024-12-09T20:42:17,878 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/A is initiating minor compaction (all files) 2024-12-09T20:42:17,878 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/B in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,878 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/A in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,879 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0453aabb3ab4466bafb6834575d65dd2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/da57747d85df470e818c4777cd69ae35, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ee7b00b3551146ef8d5916896a86f14a] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=30.6 K 2024-12-09T20:42:17,879 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d418365a90de4a94b840cc73c49ef42e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/fb95bc54a72c47f1b7770a26e3a6b798, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/41a173f92ec741d29f83f50a586bda93] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=82.3 K 2024-12-09T20:42:17,879 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,879 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d418365a90de4a94b840cc73c49ef42e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/fb95bc54a72c47f1b7770a26e3a6b798, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/41a173f92ec741d29f83f50a586bda93] 2024-12-09T20:42:17,880 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0453aabb3ab4466bafb6834575d65dd2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733776935606 2024-12-09T20:42:17,880 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d418365a90de4a94b840cc73c49ef42e, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733776935606 2024-12-09T20:42:17,880 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting da57747d85df470e818c4777cd69ae35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733776935770 2024-12-09T20:42:17,880 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting ee7b00b3551146ef8d5916896a86f14a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733776936167 2024-12-09T20:42:17,881 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb95bc54a72c47f1b7770a26e3a6b798, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733776935770 2024-12-09T20:42:17,882 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a173f92ec741d29f83f50a586bda93, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733776936152 2024-12-09T20:42:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T20:42:17,892 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#B#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:17,892 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/c1e745226a7941558f69c3ad1f6f72ca is 50, key is test_row_0/B:col10/1733776936530/Put/seqid=0 2024-12-09T20:42:17,894 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:17,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741963_1139 (size=12104) 2024-12-09T20:42:17,913 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/c1e745226a7941558f69c3ad1f6f72ca as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/c1e745226a7941558f69c3ad1f6f72ca 2024-12-09T20:42:17,918 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209f8e8a5d8c1de4715b7c8e9ea4591d55e_c7804971d6cc1c0c364cf6c88d0c46fe store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:17,920 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209f8e8a5d8c1de4715b7c8e9ea4591d55e_c7804971d6cc1c0c364cf6c88d0c46fe, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:17,921 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f8e8a5d8c1de4715b7c8e9ea4591d55e_c7804971d6cc1c0c364cf6c88d0c46fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:17,922 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/B of c7804971d6cc1c0c364cf6c88d0c46fe into c1e745226a7941558f69c3ad1f6f72ca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:17,922 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:17,922 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/B, priority=13, startTime=1733776937877; duration=0sec 2024-12-09T20:42:17,922 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:17,922 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:B 2024-12-09T20:42:17,922 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:17,925 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:17,925 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/C is initiating minor compaction (all files) 2024-12-09T20:42:17,925 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/C in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,925 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/687f51b6dd124ecbb2641730c8db765e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/e8b7ef1013f54289aeb86e8192a1b137, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f478e49650774032af2253d734de1203] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=30.6 K 2024-12-09T20:42:17,926 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 687f51b6dd124ecbb2641730c8db765e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733776935606 2024-12-09T20:42:17,929 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b7ef1013f54289aeb86e8192a1b137, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733776935770 2024-12-09T20:42:17,930 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f478e49650774032af2253d734de1203, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733776936167 2024-12-09T20:42:17,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741964_1140 (size=4469) 2024-12-09T20:42:17,941 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#C#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:17,942 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/8b10349c36cb4b84a6f5f58fb5bb9197 is 50, key is test_row_0/C:col10/1733776936530/Put/seqid=0 2024-12-09T20:42:17,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741965_1141 (size=12104) 2024-12-09T20:42:17,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T20:42:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:17,987 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-09T20:42:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:17,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209393f3b427ce541b291f288dca7e5f572_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776936643/Put/seqid=0 2024-12-09T20:42:18,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741966_1142 (size=12154) 2024-12-09T20:42:18,334 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#A#compaction#121 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:18,334 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/f848ab7d22ab46f19156c8f65dea9b0e is 175, key is test_row_0/A:col10/1733776936530/Put/seqid=0 2024-12-09T20:42:18,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741967_1143 (size=31165) 2024-12-09T20:42:18,366 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/8b10349c36cb4b84a6f5f58fb5bb9197 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8b10349c36cb4b84a6f5f58fb5bb9197 2024-12-09T20:42:18,374 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/C of c7804971d6cc1c0c364cf6c88d0c46fe into 8b10349c36cb4b84a6f5f58fb5bb9197(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:18,374 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:18,374 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/C, priority=13, startTime=1733776937877; duration=0sec 2024-12-09T20:42:18,374 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:18,374 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:C 2024-12-09T20:42:18,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:18,417 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209393f3b427ce541b291f288dca7e5f572_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209393f3b427ce541b291f288dca7e5f572_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:18,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/d65e4c37789f471fa3caa480d33d8407, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:18,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/d65e4c37789f471fa3caa480d33d8407 is 175, key is test_row_0/A:col10/1733776936643/Put/seqid=0 2024-12-09T20:42:18,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741968_1144 (size=30955) 2024-12-09T20:42:18,749 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/f848ab7d22ab46f19156c8f65dea9b0e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/f848ab7d22ab46f19156c8f65dea9b0e 2024-12-09T20:42:18,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:18,759 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/A of c7804971d6cc1c0c364cf6c88d0c46fe into f848ab7d22ab46f19156c8f65dea9b0e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:18,759 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:18,759 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/A, priority=13, startTime=1733776937877; duration=0sec 2024-12-09T20:42:18,759 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:18,760 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:A 2024-12-09T20:42:18,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776998789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:18,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733776998791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:18,793 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733776998811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:18,813 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:18,826 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/d65e4c37789f471fa3caa480d33d8407 2024-12-09T20:42:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/bcc67bb4ef454d0486c1906ebce2e42f is 50, key is test_row_0/B:col10/1733776936643/Put/seqid=0 2024-12-09T20:42:18,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741969_1145 (size=12001) 2024-12-09T20:42:18,845 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/bcc67bb4ef454d0486c1906ebce2e42f 2024-12-09T20:42:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/f93571d5f3544df58b3f886f59fc461c is 50, key is test_row_0/C:col10/1733776936643/Put/seqid=0 2024-12-09T20:42:18,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741970_1146 (size=12001) 2024-12-09T20:42:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T20:42:18,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776998901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:19,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776999111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:19,263 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/f93571d5f3544df58b3f886f59fc461c 2024-12-09T20:42:19,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/d65e4c37789f471fa3caa480d33d8407 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d65e4c37789f471fa3caa480d33d8407 2024-12-09T20:42:19,272 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d65e4c37789f471fa3caa480d33d8407, entries=150, sequenceid=97, filesize=30.2 K 2024-12-09T20:42:19,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/bcc67bb4ef454d0486c1906ebce2e42f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/bcc67bb4ef454d0486c1906ebce2e42f 2024-12-09T20:42:19,280 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/bcc67bb4ef454d0486c1906ebce2e42f, entries=150, sequenceid=97, filesize=11.7 K 2024-12-09T20:42:19,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/f93571d5f3544df58b3f886f59fc461c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f93571d5f3544df58b3f886f59fc461c 2024-12-09T20:42:19,286 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f93571d5f3544df58b3f886f59fc461c, entries=150, sequenceid=97, filesize=11.7 K 2024-12-09T20:42:19,287 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for c7804971d6cc1c0c364cf6c88d0c46fe in 1300ms, sequenceid=97, compaction requested=false 2024-12-09T20:42:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-09T20:42:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-09T20:42:19,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-09T20:42:19,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5340 sec 2024-12-09T20:42:19,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.5390 sec 2024-12-09T20:42:19,365 INFO [master/00c23a7251f8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T20:42:19,365 INFO [master/00c23a7251f8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T20:42:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:19,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:42:19,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:19,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:19,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:19,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:19,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:19,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:19,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209af3ed6372119402284a27d3bcf98395b_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776939422/Put/seqid=0 2024-12-09T20:42:19,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741971_1147 (size=14594) 2024-12-09T20:42:19,436 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:19,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209af3ed6372119402284a27d3bcf98395b_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209af3ed6372119402284a27d3bcf98395b_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:19,443 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ba6631af64834353b018957ac783a73c, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:19,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ba6631af64834353b018957ac783a73c is 175, key is test_row_0/A:col10/1733776939422/Put/seqid=0 2024-12-09T20:42:19,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741972_1148 (size=39549) 2024-12-09T20:42:19,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776999492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776999601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:19,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733776999811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:19,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ba6631af64834353b018957ac783a73c 2024-12-09T20:42:19,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51998 deadline: 1733776999862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1733776999862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:19,865 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:19,865 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:19,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f30ca23d42a8429089ca5ab48929b39c is 50, key is test_row_0/B:col10/1733776939422/Put/seqid=0 2024-12-09T20:42:19,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741973_1149 (size=12001) 2024-12-09T20:42:19,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f30ca23d42a8429089ca5ab48929b39c 2024-12-09T20:42:19,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/d0ebfc0e2984403ebd41bac3d588e4c9 is 50, key is test_row_0/C:col10/1733776939422/Put/seqid=0 2024-12-09T20:42:19,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741974_1150 (size=12001) 2024-12-09T20:42:20,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777000121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:20,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/d0ebfc0e2984403ebd41bac3d588e4c9 2024-12-09T20:42:20,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ba6631af64834353b018957ac783a73c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ba6631af64834353b018957ac783a73c 2024-12-09T20:42:20,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ba6631af64834353b018957ac783a73c, entries=200, sequenceid=111, filesize=38.6 K 2024-12-09T20:42:20,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f30ca23d42a8429089ca5ab48929b39c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f30ca23d42a8429089ca5ab48929b39c 2024-12-09T20:42:20,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f30ca23d42a8429089ca5ab48929b39c, entries=150, sequenceid=111, filesize=11.7 K 2024-12-09T20:42:20,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/d0ebfc0e2984403ebd41bac3d588e4c9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d0ebfc0e2984403ebd41bac3d588e4c9 2024-12-09T20:42:20,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d0ebfc0e2984403ebd41bac3d588e4c9, entries=150, sequenceid=111, filesize=11.7 K 2024-12-09T20:42:20,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c7804971d6cc1c0c364cf6c88d0c46fe in 891ms, sequenceid=111, compaction requested=true 2024-12-09T20:42:20,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:20,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:20,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:20,314 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:20,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:20,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:20,314 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:20,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:20,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:20,315 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:20,315 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:20,315 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/B is initiating minor compaction (all files) 2024-12-09T20:42:20,315 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/A is initiating minor compaction (all files) 2024-12-09T20:42:20,315 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/B in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:20,315 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/A in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:20,315 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/c1e745226a7941558f69c3ad1f6f72ca, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/bcc67bb4ef454d0486c1906ebce2e42f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f30ca23d42a8429089ca5ab48929b39c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=35.3 K 2024-12-09T20:42:20,315 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/f848ab7d22ab46f19156c8f65dea9b0e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d65e4c37789f471fa3caa480d33d8407, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ba6631af64834353b018957ac783a73c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=99.3 K 2024-12-09T20:42:20,315 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:20,315 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/f848ab7d22ab46f19156c8f65dea9b0e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d65e4c37789f471fa3caa480d33d8407, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ba6631af64834353b018957ac783a73c] 2024-12-09T20:42:20,316 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c1e745226a7941558f69c3ad1f6f72ca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733776935770 2024-12-09T20:42:20,316 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting bcc67bb4ef454d0486c1906ebce2e42f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733776936622 2024-12-09T20:42:20,316 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting f848ab7d22ab46f19156c8f65dea9b0e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733776935770 2024-12-09T20:42:20,316 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f30ca23d42a8429089ca5ab48929b39c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733776938777 2024-12-09T20:42:20,316 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d65e4c37789f471fa3caa480d33d8407, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733776936622 2024-12-09T20:42:20,317 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba6631af64834353b018957ac783a73c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733776938777 2024-12-09T20:42:20,323 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:20,324 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#B#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:20,325 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/7718d2f826c9494e8a7f9a0f30837fc3 is 50, key is test_row_0/B:col10/1733776939422/Put/seqid=0 2024-12-09T20:42:20,326 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209b5e2144b7ce94906b85ecce70d076f74_c7804971d6cc1c0c364cf6c88d0c46fe store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:20,327 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209b5e2144b7ce94906b85ecce70d076f74_c7804971d6cc1c0c364cf6c88d0c46fe, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:20,327 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b5e2144b7ce94906b85ecce70d076f74_c7804971d6cc1c0c364cf6c88d0c46fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:20,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741976_1152 (size=4469) 2024-12-09T20:42:20,346 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#A#compaction#130 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:20,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741975_1151 (size=12207) 2024-12-09T20:42:20,347 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/9fe589408b1b422d9f5f6b897e86220f is 175, key is test_row_0/A:col10/1733776939422/Put/seqid=0 2024-12-09T20:42:20,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741977_1153 (size=31161) 2024-12-09T20:42:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:20,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776939482, max timestamp=1733776940631Num uniques 859; 2024-12-09T20:42:20,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776939482, max timestamp=1733776940631Num uniques 859; 2024-12-09T20:42:20,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776939482, max timestamp=1733776940631Num uniques 859; 2024-12-09T20:42:20,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:20,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:20,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:20,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-12-09T20:42:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:20,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092a85ee51ae084b3780600ef46be2208e_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776940631/Put/seqid=0 2024-12-09T20:42:20,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741978_1154 (size=17234) 2024-12-09T20:42:20,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776940664, max timestamp=1733776940711Num uniques 859; 2024-12-09T20:42:20,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:20,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776940664, max timestamp=1733776940711Num uniques 859; 2024-12-09T20:42:20,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776940664, max timestamp=1733776940711Num uniques 859; 2024-12-09T20:42:20,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:20,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:20,752 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/7718d2f826c9494e8a7f9a0f30837fc3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/7718d2f826c9494e8a7f9a0f30837fc3 2024-12-09T20:42:20,758 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/B of c7804971d6cc1c0c364cf6c88d0c46fe into 7718d2f826c9494e8a7f9a0f30837fc3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:20,758 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:20,758 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/B, priority=13, startTime=1733776940314; duration=0sec 2024-12-09T20:42:20,758 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:20,758 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:B 2024-12-09T20:42:20,758 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:20,762 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:20,762 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/C is initiating minor compaction (all files) 2024-12-09T20:42:20,763 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/C in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:20,763 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8b10349c36cb4b84a6f5f58fb5bb9197, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f93571d5f3544df58b3f886f59fc461c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d0ebfc0e2984403ebd41bac3d588e4c9] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=35.3 K 2024-12-09T20:42:20,763 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b10349c36cb4b84a6f5f58fb5bb9197, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733776935770 2024-12-09T20:42:20,763 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f93571d5f3544df58b3f886f59fc461c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733776936622 2024-12-09T20:42:20,764 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/9fe589408b1b422d9f5f6b897e86220f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/9fe589408b1b422d9f5f6b897e86220f 2024-12-09T20:42:20,764 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d0ebfc0e2984403ebd41bac3d588e4c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733776938777 2024-12-09T20:42:20,770 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/A of c7804971d6cc1c0c364cf6c88d0c46fe into 9fe589408b1b422d9f5f6b897e86220f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:20,770 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:20,770 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/A, priority=13, startTime=1733776940314; duration=0sec 2024-12-09T20:42:20,770 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:20,770 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:A 2024-12-09T20:42:20,775 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:20,775 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/5833b527770a4fdab24a84210f954c3a is 50, key is test_row_0/C:col10/1733776939422/Put/seqid=0 2024-12-09T20:42:20,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741979_1155 (size=12207) 2024-12-09T20:42:20,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733776940754, max timestamp=1733776940782Num uniques 1018; 2024-12-09T20:42:20,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T20:42:20,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733776940754, max timestamp=1733776940782Num uniques 1018; 2024-12-09T20:42:20,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T20:42:20,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733776940754, max timestamp=1733776940782Num uniques 1018; 2024-12-09T20:42:20,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T20:42:20,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777000831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:20,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733776940814, max timestamp=1733776940829Num uniques 1077; 2024-12-09T20:42:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:20,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-09T20:42:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:20,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733776940814, max timestamp=1733776940829Num uniques 1077; 2024-12-09T20:42:20,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-09T20:42:20,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733776940814, max timestamp=1733776940829Num uniques 1077; 2024-12-09T20:42:20,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-09T20:42:20,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733776940862, max timestamp=1733776940872Num uniques 1036; 2024-12-09T20:42:20,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733776940862, max timestamp=1733776940872Num uniques 1036; 2024-12-09T20:42:20,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=8506, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T20:42:20,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=8506, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T20:42:20,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733776940862, max timestamp=1733776940872Num uniques 1036; 2024-12-09T20:42:20,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=8506, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T20:42:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T20:42:20,911 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:20,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-09T20:42:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T20:42:20,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:20,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:20,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:20,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733776940901, max timestamp=1733776940919Num uniques 1045; 2024-12-09T20:42:20,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1733776940901, max timestamp=1733776940919Num uniques 1045; 2024-12-09T20:42:20,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T20:42:20,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=8920, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T20:42:20,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1733776940901, max timestamp=1733776940919Num uniques 1045; 2024-12-09T20:42:20,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=8920, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T20:42:20,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733776940942, max timestamp=1733776940965Num uniques 1054; 2024-12-09T20:42:20,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733776940942, max timestamp=1733776940965Num uniques 1054; 2024-12-09T20:42:20,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733776940942, max timestamp=1733776940965Num uniques 1054; 2024-12-09T20:42:20,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-09T20:42:20,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-09T20:42:20,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-09T20:42:21,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777000998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1733776940978, max timestamp=1733776941001Num uniques 1113; 2024-12-09T20:42:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777001000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1733776940978, max timestamp=1733776941001Num uniques 1113; 2024-12-09T20:42:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1733776940978, max timestamp=1733776941001Num uniques 1113; 2024-12-09T20:42:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=16618, suffix heap size=143720,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=363 2024-12-09T20:42:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=16618, suffix heap size=143720,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=363 2024-12-09T20:42:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=16618, suffix heap size=143720,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=363 2024-12-09T20:42:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T20:42:21,057 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:21,062 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092a85ee51ae084b3780600ef46be2208e_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092a85ee51ae084b3780600ef46be2208e_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:21,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/4db8ae316ad240e7a77676d97d7dd210, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:21,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/4db8ae316ad240e7a77676d97d7dd210 is 175, key is test_row_0/A:col10/1733776940631/Put/seqid=0 2024-12-09T20:42:21,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741980_1156 (size=48339) 2024-12-09T20:42:21,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733777001087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777001111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777001111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,184 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/5833b527770a4fdab24a84210f954c3a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/5833b527770a4fdab24a84210f954c3a 2024-12-09T20:42:21,191 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/C of c7804971d6cc1c0c364cf6c88d0c46fe into 5833b527770a4fdab24a84210f954c3a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:21,191 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:21,191 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/C, priority=13, startTime=1733776940314; duration=0sec 2024-12-09T20:42:21,191 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:21,191 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:C 2024-12-09T20:42:21,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733777001191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T20:42:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777001321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777001321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733777001401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/4db8ae316ad240e7a77676d97d7dd210 2024-12-09T20:42:21,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f5bb4aa15acb4da8b0b5817a3c657e73 is 50, key is test_row_0/B:col10/1733776940631/Put/seqid=0 2024-12-09T20:42:21,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741981_1157 (size=12151) 2024-12-09T20:42:21,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T20:42:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777001631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777001631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733777001711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:21,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:21,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:21,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f5bb4aa15acb4da8b0b5817a3c657e73 2024-12-09T20:42:21,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/c29337ff08594d4fbe08afb4dcd585ee is 50, key is test_row_0/C:col10/1733776940631/Put/seqid=0 2024-12-09T20:42:21,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741982_1158 (size=12151) 2024-12-09T20:42:21,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:21,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T20:42:22,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:22,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777002141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777002141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:22,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733777002221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:22,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:22,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:22,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/c29337ff08594d4fbe08afb4dcd585ee 2024-12-09T20:42:22,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/4db8ae316ad240e7a77676d97d7dd210 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4db8ae316ad240e7a77676d97d7dd210 2024-12-09T20:42:22,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4db8ae316ad240e7a77676d97d7dd210, entries=250, sequenceid=133, filesize=47.2 K 2024-12-09T20:42:22,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f5bb4aa15acb4da8b0b5817a3c657e73 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5bb4aa15acb4da8b0b5817a3c657e73 2024-12-09T20:42:22,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5bb4aa15acb4da8b0b5817a3c657e73, entries=150, sequenceid=133, filesize=11.9 K 2024-12-09T20:42:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/c29337ff08594d4fbe08afb4dcd585ee as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/c29337ff08594d4fbe08afb4dcd585ee 2024-12-09T20:42:22,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/c29337ff08594d4fbe08afb4dcd585ee, entries=150, sequenceid=133, filesize=11.9 K 2024-12-09T20:42:22,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for c7804971d6cc1c0c364cf6c88d0c46fe in 1699ms, sequenceid=133, compaction requested=false 2024-12-09T20:42:22,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:22,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T20:42:22,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:22,445 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=161.02 KB heapSize=445.55 KB 2024-12-09T20:42:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:22,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209059a6195c4a5429c8c8f79e1d09be8c0_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776941054/Put/seqid=0 2024-12-09T20:42:22,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741983_1159 (size=12454) 2024-12-09T20:42:22,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:22,465 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209059a6195c4a5429c8c8f79e1d09be8c0_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209059a6195c4a5429c8c8f79e1d09be8c0_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0dd099144b5c429eb8e7108f5ac82046, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0dd099144b5c429eb8e7108f5ac82046 is 175, key is test_row_0/A:col10/1733776941054/Put/seqid=0 2024-12-09T20:42:22,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741984_1160 (size=31255) 2024-12-09T20:42:22,873 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0dd099144b5c429eb8e7108f5ac82046 2024-12-09T20:42:22,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/63f999f652d246ad82fd106a166afac2 is 50, key is test_row_0/B:col10/1733776941054/Put/seqid=0 2024-12-09T20:42:22,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741985_1161 (size=12301) 2024-12-09T20:42:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T20:42:23,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:23,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777003159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777003161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733777003241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,241 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777003271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777003271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,285 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/63f999f652d246ad82fd106a166afac2 2024-12-09T20:42:23,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/cf5486ed07344fa5b7930822724ff8e4 is 50, key is test_row_0/C:col10/1733776941054/Put/seqid=0 2024-12-09T20:42:23,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741986_1162 (size=12301) 2024-12-09T20:42:23,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777003481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777003481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,698 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/cf5486ed07344fa5b7930822724ff8e4 2024-12-09T20:42:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0dd099144b5c429eb8e7108f5ac82046 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0dd099144b5c429eb8e7108f5ac82046 2024-12-09T20:42:23,707 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0dd099144b5c429eb8e7108f5ac82046, entries=150, sequenceid=276, filesize=30.5 K 2024-12-09T20:42:23,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/63f999f652d246ad82fd106a166afac2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/63f999f652d246ad82fd106a166afac2 2024-12-09T20:42:23,712 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/63f999f652d246ad82fd106a166afac2, entries=150, sequenceid=276, filesize=12.0 K 2024-12-09T20:42:23,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/cf5486ed07344fa5b7930822724ff8e4 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/cf5486ed07344fa5b7930822724ff8e4 2024-12-09T20:42:23,717 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/cf5486ed07344fa5b7930822724ff8e4, entries=150, sequenceid=276, filesize=12.0 K 2024-12-09T20:42:23,718 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.50 KB/456192, currentSize=26.84 KB/27480 for c7804971d6cc1c0c364cf6c88d0c46fe in 1273ms, sequenceid=276, compaction requested=true 2024-12-09T20:42:23,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:23,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:23,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-09T20:42:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-09T20:42:23,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-09T20:42:23,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8060 sec 2024-12-09T20:42:23,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.8100 sec 2024-12-09T20:42:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:23,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:42:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:23,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab6b4e60fe03453fb2d167facc196582_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776943796/Put/seqid=0 2024-12-09T20:42:23,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741987_1163 (size=17534) 2024-12-09T20:42:23,808 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:23,813 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab6b4e60fe03453fb2d167facc196582_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab6b4e60fe03453fb2d167facc196582_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:23,814 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/19c51d42fbe44bff93a770437b0e904e, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:23,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/19c51d42fbe44bff93a770437b0e904e is 175, key is test_row_0/A:col10/1733776943796/Put/seqid=0 2024-12-09T20:42:23,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741988_1164 (size=48639) 2024-12-09T20:42:23,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733776943823, max timestamp=1733776943835Num uniques 859; 2024-12-09T20:42:23,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733776943823, max timestamp=1733776943835Num uniques 859; 2024-12-09T20:42:23,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:42:23,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:42:23,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733776943823, max timestamp=1733776943835Num uniques 859; 2024-12-09T20:42:23,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:42:23,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777003863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777003869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1733777003871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,874 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51998 deadline: 1733777003901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,902 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:23,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777003971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777003981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:24,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777004181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:24,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777004191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:24,220 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/19c51d42fbe44bff93a770437b0e904e 2024-12-09T20:42:24,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/ec0af045ade44747a7702c27ec9b437d is 50, key is test_row_0/B:col10/1733776943796/Put/seqid=0 2024-12-09T20:42:24,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741989_1165 (size=12301) 2024-12-09T20:42:24,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/ec0af045ade44747a7702c27ec9b437d 2024-12-09T20:42:24,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/2092b2b84f6a423181933a2a539b587a is 50, key is test_row_0/C:col10/1733776943796/Put/seqid=0 2024-12-09T20:42:24,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741990_1166 (size=12301) 2024-12-09T20:42:24,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777004491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777004501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:24,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/2092b2b84f6a423181933a2a539b587a 2024-12-09T20:42:24,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/19c51d42fbe44bff93a770437b0e904e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/19c51d42fbe44bff93a770437b0e904e 2024-12-09T20:42:24,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/19c51d42fbe44bff93a770437b0e904e, entries=250, sequenceid=287, filesize=47.5 K 2024-12-09T20:42:24,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/ec0af045ade44747a7702c27ec9b437d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ec0af045ade44747a7702c27ec9b437d 2024-12-09T20:42:24,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ec0af045ade44747a7702c27ec9b437d, entries=150, sequenceid=287, filesize=12.0 K 2024-12-09T20:42:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/2092b2b84f6a423181933a2a539b587a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/2092b2b84f6a423181933a2a539b587a 2024-12-09T20:42:24,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/2092b2b84f6a423181933a2a539b587a, entries=150, sequenceid=287, filesize=12.0 K 2024-12-09T20:42:24,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for c7804971d6cc1c0c364cf6c88d0c46fe in 884ms, sequenceid=287, compaction requested=true 2024-12-09T20:42:24,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:24,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:24,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:24,681 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:24,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:24,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:24,681 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:24,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:24,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:24,683 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48960 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:24,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:24,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/A is initiating minor compaction (all files) 2024-12-09T20:42:24,683 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/B is initiating minor compaction (all files) 2024-12-09T20:42:24,683 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/B in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:24,683 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/A in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:24,683 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/7718d2f826c9494e8a7f9a0f30837fc3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5bb4aa15acb4da8b0b5817a3c657e73, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/63f999f652d246ad82fd106a166afac2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ec0af045ade44747a7702c27ec9b437d] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=47.8 K 2024-12-09T20:42:24,683 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/9fe589408b1b422d9f5f6b897e86220f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4db8ae316ad240e7a77676d97d7dd210, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0dd099144b5c429eb8e7108f5ac82046, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/19c51d42fbe44bff93a770437b0e904e] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=155.7 K 2024-12-09T20:42:24,683 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:24,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/9fe589408b1b422d9f5f6b897e86220f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4db8ae316ad240e7a77676d97d7dd210, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0dd099144b5c429eb8e7108f5ac82046, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/19c51d42fbe44bff93a770437b0e904e] 2024-12-09T20:42:24,683 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 7718d2f826c9494e8a7f9a0f30837fc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733776938777 2024-12-09T20:42:24,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fe589408b1b422d9f5f6b897e86220f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733776938777 2024-12-09T20:42:24,684 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4db8ae316ad240e7a77676d97d7dd210, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733776939482 2024-12-09T20:42:24,684 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f5bb4aa15acb4da8b0b5817a3c657e73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733776939484 2024-12-09T20:42:24,684 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd099144b5c429eb8e7108f5ac82046, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733776941037 2024-12-09T20:42:24,684 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f999f652d246ad82fd106a166afac2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733776941037 2024-12-09T20:42:24,684 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c51d42fbe44bff93a770437b0e904e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733776943158 2024-12-09T20:42:24,684 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting ec0af045ade44747a7702c27ec9b437d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733776943792 2024-12-09T20:42:24,695 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:24,696 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#B#compaction#141 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:24,696 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/e71d01720162470396ee363546540858 is 50, key is test_row_0/B:col10/1733776943796/Put/seqid=0 2024-12-09T20:42:24,698 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412090bd6762352e84248a15fb097a59ab74b_c7804971d6cc1c0c364cf6c88d0c46fe store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:24,701 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412090bd6762352e84248a15fb097a59ab74b_c7804971d6cc1c0c364cf6c88d0c46fe, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:24,701 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090bd6762352e84248a15fb097a59ab74b_c7804971d6cc1c0c364cf6c88d0c46fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:24,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741991_1167 (size=12643) 2024-12-09T20:42:24,713 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/e71d01720162470396ee363546540858 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e71d01720162470396ee363546540858 2024-12-09T20:42:24,718 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/B of c7804971d6cc1c0c364cf6c88d0c46fe into e71d01720162470396ee363546540858(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:24,719 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:24,719 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/B, priority=12, startTime=1733776944681; duration=0sec 2024-12-09T20:42:24,719 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:24,719 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:B 2024-12-09T20:42:24,719 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:24,722 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48960 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:24,722 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/C is initiating minor compaction (all files) 2024-12-09T20:42:24,722 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/C in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:24,722 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/5833b527770a4fdab24a84210f954c3a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/c29337ff08594d4fbe08afb4dcd585ee, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/cf5486ed07344fa5b7930822724ff8e4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/2092b2b84f6a423181933a2a539b587a] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=47.8 K 2024-12-09T20:42:24,723 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 5833b527770a4fdab24a84210f954c3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733776938777 2024-12-09T20:42:24,723 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c29337ff08594d4fbe08afb4dcd585ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733776939484 2024-12-09T20:42:24,723 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting cf5486ed07344fa5b7930822724ff8e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733776941037 2024-12-09T20:42:24,723 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2092b2b84f6a423181933a2a539b587a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733776943792 2024-12-09T20:42:24,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741992_1168 (size=4469) 2024-12-09T20:42:24,735 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#C#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:24,736 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/6c67e60f5821418e9dca30193079c228 is 50, key is test_row_0/C:col10/1733776943796/Put/seqid=0 2024-12-09T20:42:24,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741993_1169 (size=12643) 2024-12-09T20:42:24,744 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/6c67e60f5821418e9dca30193079c228 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6c67e60f5821418e9dca30193079c228 2024-12-09T20:42:24,749 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/C of c7804971d6cc1c0c364cf6c88d0c46fe into 6c67e60f5821418e9dca30193079c228(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:24,749 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:24,749 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/C, priority=12, startTime=1733776944681; duration=0sec 2024-12-09T20:42:24,749 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:24,749 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:C 2024-12-09T20:42:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:25,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-09T20:42:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:25,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093761585be0204aa1b154880cc68d8521_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776945001/Put/seqid=0 2024-12-09T20:42:25,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741994_1170 (size=14994) 2024-12-09T20:42:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T20:42:25,071 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:25,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-09T20:42:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T20:42:25,073 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:25,074 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:25,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:25,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777005074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777005075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:25,131 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:25,132 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0951dfade88a4ae59481bbabc4b08fa1 is 175, key is test_row_0/A:col10/1733776943796/Put/seqid=0 2024-12-09T20:42:25,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741995_1171 (size=31597) 2024-12-09T20:42:25,141 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0951dfade88a4ae59481bbabc4b08fa1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0951dfade88a4ae59481bbabc4b08fa1 2024-12-09T20:42:25,145 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/A of c7804971d6cc1c0c364cf6c88d0c46fe into 0951dfade88a4ae59481bbabc4b08fa1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:25,145 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:25,145 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/A, priority=12, startTime=1733776944681; duration=0sec 2024-12-09T20:42:25,145 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:25,145 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:A 2024-12-09T20:42:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T20:42:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777005181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:25,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777005191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:25,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733777005271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:25,273 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:25,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T20:42:25,372 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T20:42:25,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:25,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:25,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T20:42:25,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777005391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:25,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777005400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:25,468 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:25,473 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093761585be0204aa1b154880cc68d8521_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093761585be0204aa1b154880cc68d8521_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:25,474 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/91ffa3c840bf4dba84039ebb6c49e2b4, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:25,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/91ffa3c840bf4dba84039ebb6c49e2b4 is 175, key is test_row_0/A:col10/1733776945001/Put/seqid=0 2024-12-09T20:42:25,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741996_1172 (size=39949) 2024-12-09T20:42:25,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T20:42:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777005701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:25,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777005711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:25,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,888 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/91ffa3c840bf4dba84039ebb6c49e2b4 2024-12-09T20:42:25,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/3e6aaa17a62d41dba23d12fc25491d92 is 50, key is test_row_0/B:col10/1733776945001/Put/seqid=0 2024-12-09T20:42:25,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741997_1173 (size=12301) 2024-12-09T20:42:25,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:25,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:26,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:26,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:26,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:26,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T20:42:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777006211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:26,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777006221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:26,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:26,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/3e6aaa17a62d41dba23d12fc25491d92 2024-12-09T20:42:26,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/d6f662914ab647879257a83bb39587f2 is 50, key is test_row_0/C:col10/1733776945001/Put/seqid=0 2024-12-09T20:42:26,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741998_1174 (size=12301) 2024-12-09T20:42:26,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:26,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:26,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:26,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:26,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:26,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/d6f662914ab647879257a83bb39587f2 2024-12-09T20:42:26,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/91ffa3c840bf4dba84039ebb6c49e2b4 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/91ffa3c840bf4dba84039ebb6c49e2b4 2024-12-09T20:42:26,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/91ffa3c840bf4dba84039ebb6c49e2b4, entries=200, sequenceid=324, filesize=39.0 K 2024-12-09T20:42:26,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/3e6aaa17a62d41dba23d12fc25491d92 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/3e6aaa17a62d41dba23d12fc25491d92 2024-12-09T20:42:26,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/3e6aaa17a62d41dba23d12fc25491d92, entries=150, sequenceid=324, filesize=12.0 K 2024-12-09T20:42:26,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/d6f662914ab647879257a83bb39587f2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d6f662914ab647879257a83bb39587f2 2024-12-09T20:42:26,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d6f662914ab647879257a83bb39587f2, entries=150, sequenceid=324, filesize=12.0 K 2024-12-09T20:42:26,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for c7804971d6cc1c0c364cf6c88d0c46fe in 1734ms, sequenceid=324, compaction requested=false 2024-12-09T20:42:26,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:26,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T20:42:26,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:26,758 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:42:26,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:26,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:26,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:26,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:26,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:26,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:26,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120951f17abf39f541209ca3119545a6a90a_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776945006/Put/seqid=0 2024-12-09T20:42:26,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741999_1175 (size=12454) 2024-12-09T20:42:27,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:27,180 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120951f17abf39f541209ca3119545a6a90a_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120951f17abf39f541209ca3119545a6a90a_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ed60108a59f4480ea1167deed583428e, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ed60108a59f4480ea1167deed583428e is 175, key is test_row_0/A:col10/1733776945006/Put/seqid=0 2024-12-09T20:42:27,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742000_1176 (size=31255) 2024-12-09T20:42:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T20:42:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:27,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:27,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733776947251, max timestamp=1733776947261Num uniques 859; 2024-12-09T20:42:27,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733776947251, max timestamp=1733776947261Num uniques 859; 2024-12-09T20:42:27,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T20:42:27,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T20:42:27,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733776947251, max timestamp=1733776947261Num uniques 859; 2024-12-09T20:42:27,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T20:42:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777007302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777007307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:27,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777007411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777007411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:27,586 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ed60108a59f4480ea1167deed583428e 2024-12-09T20:42:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/03363229df004a3e9c5fd9f194e1d06b is 50, key is test_row_0/B:col10/1733776945006/Put/seqid=0 2024-12-09T20:42:27,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742001_1177 (size=12301) 2024-12-09T20:42:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777007621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777007621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777007931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777007931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:27,997 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/03363229df004a3e9c5fd9f194e1d06b 2024-12-09T20:42:28,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/9a3ee2c8bee448e3b7c47406f6a94dbf is 50, key is test_row_0/C:col10/1733776945006/Put/seqid=0 2024-12-09T20:42:28,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742002_1178 (size=12301) 2024-12-09T20:42:28,411 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/9a3ee2c8bee448e3b7c47406f6a94dbf 2024-12-09T20:42:28,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ed60108a59f4480ea1167deed583428e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ed60108a59f4480ea1167deed583428e 2024-12-09T20:42:28,420 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ed60108a59f4480ea1167deed583428e, entries=150, sequenceid=335, filesize=30.5 K 2024-12-09T20:42:28,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/03363229df004a3e9c5fd9f194e1d06b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/03363229df004a3e9c5fd9f194e1d06b 2024-12-09T20:42:28,425 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/03363229df004a3e9c5fd9f194e1d06b, entries=150, sequenceid=335, filesize=12.0 K 2024-12-09T20:42:28,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/9a3ee2c8bee448e3b7c47406f6a94dbf as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/9a3ee2c8bee448e3b7c47406f6a94dbf 2024-12-09T20:42:28,430 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/9a3ee2c8bee448e3b7c47406f6a94dbf, entries=150, sequenceid=335, filesize=12.0 K 2024-12-09T20:42:28,431 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c7804971d6cc1c0c364cf6c88d0c46fe in 1673ms, sequenceid=335, compaction requested=true 2024-12-09T20:42:28,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:28,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:28,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-09T20:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-09T20:42:28,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-09T20:42:28,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3590 sec 2024-12-09T20:42:28,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.3640 sec 2024-12-09T20:42:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:28,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-09T20:42:28,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:28,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:28,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:28,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:28,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:28,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:28,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209332e84e0d1434bdbb10278ef70e1f718_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776947282/Put/seqid=0 2024-12-09T20:42:28,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742003_1179 (size=17534) 2024-12-09T20:42:28,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777008456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:28,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777008457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:28,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777008560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:28,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777008571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:28,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777008771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:28,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777008780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:28,854 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:28,858 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209332e84e0d1434bdbb10278ef70e1f718_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209332e84e0d1434bdbb10278ef70e1f718_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:28,859 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/b97b894e263e4a48a53c9779afceadc6, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:28,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/b97b894e263e4a48a53c9779afceadc6 is 175, key is test_row_0/A:col10/1733776947282/Put/seqid=0 2024-12-09T20:42:28,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742004_1180 (size=48639) 2024-12-09T20:42:28,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/b97b894e263e4a48a53c9779afceadc6 2024-12-09T20:42:28,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f87035f9017b431ea2e7a1f46a6b09ef is 50, key is test_row_0/B:col10/1733776947282/Put/seqid=0 2024-12-09T20:42:28,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742005_1181 (size=12301) 2024-12-09T20:42:28,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f87035f9017b431ea2e7a1f46a6b09ef 2024-12-09T20:42:28,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/19143828d74a4a42ac8781b99486a3fe is 50, key is test_row_0/C:col10/1733776947282/Put/seqid=0 2024-12-09T20:42:28,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742006_1182 (size=12301) 2024-12-09T20:42:29,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777009081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:29,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777009091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T20:42:29,241 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:29,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-09T20:42:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T20:42:29,244 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:29,244 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:29,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:29,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/19143828d74a4a42ac8781b99486a3fe 2024-12-09T20:42:29,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/b97b894e263e4a48a53c9779afceadc6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/b97b894e263e4a48a53c9779afceadc6 2024-12-09T20:42:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52002 deadline: 1733777009311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:29,313 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:29,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/b97b894e263e4a48a53c9779afceadc6, entries=250, sequenceid=372, filesize=47.5 K 2024-12-09T20:42:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f87035f9017b431ea2e7a1f46a6b09ef as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f87035f9017b431ea2e7a1f46a6b09ef 2024-12-09T20:42:29,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f87035f9017b431ea2e7a1f46a6b09ef, entries=150, sequenceid=372, filesize=12.0 K 2024-12-09T20:42:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/19143828d74a4a42ac8781b99486a3fe as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/19143828d74a4a42ac8781b99486a3fe 2024-12-09T20:42:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/19143828d74a4a42ac8781b99486a3fe, entries=150, sequenceid=372, filesize=12.0 K 2024-12-09T20:42:29,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for c7804971d6cc1c0c364cf6c88d0c46fe in 884ms, sequenceid=372, compaction requested=true 2024-12-09T20:42:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:29,327 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:29,327 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:29,329 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:29,329 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/B is initiating minor compaction (all files) 2024-12-09T20:42:29,329 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/B in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:29,329 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e71d01720162470396ee363546540858, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/3e6aaa17a62d41dba23d12fc25491d92, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/03363229df004a3e9c5fd9f194e1d06b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f87035f9017b431ea2e7a1f46a6b09ef] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=48.4 K 2024-12-09T20:42:29,329 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:29,329 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/A is initiating minor compaction (all files) 2024-12-09T20:42:29,329 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/A in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:29,329 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting e71d01720162470396ee363546540858, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733776943792 2024-12-09T20:42:29,329 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0951dfade88a4ae59481bbabc4b08fa1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/91ffa3c840bf4dba84039ebb6c49e2b4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ed60108a59f4480ea1167deed583428e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/b97b894e263e4a48a53c9779afceadc6] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=147.9 K 2024-12-09T20:42:29,330 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:29,330 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0951dfade88a4ae59481bbabc4b08fa1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/91ffa3c840bf4dba84039ebb6c49e2b4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ed60108a59f4480ea1167deed583428e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/b97b894e263e4a48a53c9779afceadc6] 2024-12-09T20:42:29,330 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e6aaa17a62d41dba23d12fc25491d92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733776943858 2024-12-09T20:42:29,330 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0951dfade88a4ae59481bbabc4b08fa1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733776943792 2024-12-09T20:42:29,330 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 03363229df004a3e9c5fd9f194e1d06b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733776945006 2024-12-09T20:42:29,330 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91ffa3c840bf4dba84039ebb6c49e2b4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733776943858 2024-12-09T20:42:29,330 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f87035f9017b431ea2e7a1f46a6b09ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733776947282 2024-12-09T20:42:29,330 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed60108a59f4480ea1167deed583428e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733776945006 2024-12-09T20:42:29,331 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting b97b894e263e4a48a53c9779afceadc6, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733776947282 2024-12-09T20:42:29,336 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,346 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#B#compaction#154 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,347 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/e63d0ac599464ed5aa1fc7df911ac462 is 50, key is test_row_0/B:col10/1733776947282/Put/seqid=0 2024-12-09T20:42:29,348 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120974cd6cc41b554fac9fe4d86e991efe0c_c7804971d6cc1c0c364cf6c88d0c46fe store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T20:42:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,358 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120974cd6cc41b554fac9fe4d86e991efe0c_c7804971d6cc1c0c364cf6c88d0c46fe, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:29,359 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120974cd6cc41b554fac9fe4d86e991efe0c_c7804971d6cc1c0c364cf6c88d0c46fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742007_1183 (size=12779) 2024-12-09T20:42:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,369 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/e63d0ac599464ed5aa1fc7df911ac462 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e63d0ac599464ed5aa1fc7df911ac462 2024-12-09T20:42:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,376 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/B of c7804971d6cc1c0c364cf6c88d0c46fe into e63d0ac599464ed5aa1fc7df911ac462(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,376 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:29,376 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/B, priority=12, startTime=1733776949327; duration=0sec 2024-12-09T20:42:29,376 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,376 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:B 2024-12-09T20:42:29,377 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,378 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:29,378 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/C is initiating minor compaction (all files) 2024-12-09T20:42:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,378 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/C in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:29,378 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6c67e60f5821418e9dca30193079c228, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d6f662914ab647879257a83bb39587f2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/9a3ee2c8bee448e3b7c47406f6a94dbf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/19143828d74a4a42ac8781b99486a3fe] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=48.4 K 2024-12-09T20:42:29,379 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c67e60f5821418e9dca30193079c228, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733776943792 2024-12-09T20:42:29,379 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d6f662914ab647879257a83bb39587f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733776943858 2024-12-09T20:42:29,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742008_1184 (size=4469) 2024-12-09T20:42:29,380 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a3ee2c8bee448e3b7c47406f6a94dbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733776945006 2024-12-09T20:42:29,380 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 19143828d74a4a42ac8781b99486a3fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733776947282 2024-12-09T20:42:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,390 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#A#compaction#153 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,391 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/967d0b45e9504831b3e913156b2cad40 is 175, key is test_row_0/A:col10/1733776947282/Put/seqid=0 2024-12-09T20:42:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T20:42:29,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:29,397 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T20:42:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:29,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,410 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#C#compaction#155 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,410 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/d3a88631cefa483185674067ecf322a7 is 50, key is test_row_0/C:col10/1733776947282/Put/seqid=0 2024-12-09T20:42:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742009_1185 (size=31733) 2024-12-09T20:42:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209fcbd89efd429452fa2b0e4d16e2f779e_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776948456/Put/seqid=0 2024-12-09T20:42:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742010_1186 (size=12779) 2024-12-09T20:42:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742011_1187 (size=9914) 2024-12-09T20:42:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,453 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209fcbd89efd429452fa2b0e4d16e2f779e_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209fcbd89efd429452fa2b0e4d16e2f779e_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:29,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/e78f4a77b90241a8a14757787daf410e, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:29,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/e78f4a77b90241a8a14757787daf410e is 175, key is test_row_0/A:col10/1733776948456/Put/seqid=0 2024-12-09T20:42:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742012_1188 (size=22561) 2024-12-09T20:42:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,468 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/e78f4a77b90241a8a14757787daf410e 2024-12-09T20:42:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/561690c2ec034f17894421037e693cb5 is 50, key is test_row_0/B:col10/1733776948456/Put/seqid=0 2024-12-09T20:42:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742013_1189 (size=9857) 2024-12-09T20:42:29,494 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/561690c2ec034f17894421037e693cb5 2024-12-09T20:42:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/41590ad2d44d4fa382e07f0bb55883a0 is 50, key is test_row_0/C:col10/1733776948456/Put/seqid=0 2024-12-09T20:42:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742014_1190 (size=9857) 2024-12-09T20:42:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,533 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/41590ad2d44d4fa382e07f0bb55883a0 2024-12-09T20:42:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/e78f4a77b90241a8a14757787daf410e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/e78f4a77b90241a8a14757787daf410e 2024-12-09T20:42:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,545 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/e78f4a77b90241a8a14757787daf410e, entries=100, sequenceid=381, filesize=22.0 K 2024-12-09T20:42:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/561690c2ec034f17894421037e693cb5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/561690c2ec034f17894421037e693cb5 2024-12-09T20:42:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,556 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/561690c2ec034f17894421037e693cb5, entries=100, sequenceid=381, filesize=9.6 K 2024-12-09T20:42:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/41590ad2d44d4fa382e07f0bb55883a0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/41590ad2d44d4fa382e07f0bb55883a0 2024-12-09T20:42:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T20:42:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,563 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/41590ad2d44d4fa382e07f0bb55883a0, entries=100, sequenceid=381, filesize=9.6 K 2024-12-09T20:42:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,564 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c7804971d6cc1c0c364cf6c88d0c46fe in 167ms, sequenceid=381, compaction requested=false 2024-12-09T20:42:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-09T20:42:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-09T20:42:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-09T20:42:29,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 321 msec 2024-12-09T20:42:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 327 msec 2024-12-09T20:42:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:29,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:42:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:29,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:29,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:29,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:29,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:29,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209593dea31f5494c3ebb639faf5686f63b_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776949653/Put/seqid=0 2024-12-09T20:42:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742015_1191 (size=22618) 2024-12-09T20:42:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:29,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733776949714, max timestamp=1733776949732Num uniques 859; 2024-12-09T20:42:29,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733776949714, max timestamp=1733776949732Num uniques 859; 2024-12-09T20:42:29,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733776949714, max timestamp=1733776949732Num uniques 859; 2024-12-09T20:42:29,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:42:29,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:42:29,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:42:29,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777009767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777009768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:29,835 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/967d0b45e9504831b3e913156b2cad40 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/967d0b45e9504831b3e913156b2cad40 2024-12-09T20:42:29,843 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/d3a88631cefa483185674067ecf322a7 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d3a88631cefa483185674067ecf322a7 2024-12-09T20:42:29,848 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/A of c7804971d6cc1c0c364cf6c88d0c46fe into 967d0b45e9504831b3e913156b2cad40(size=31.0 K), total size for store is 53.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:29,848 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:29,848 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/A, priority=12, startTime=1733776949327; duration=0sec 2024-12-09T20:42:29,848 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:29,848 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:A 2024-12-09T20:42:29,853 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/C of c7804971d6cc1c0c364cf6c88d0c46fe into d3a88631cefa483185674067ecf322a7(size=12.5 K), total size for store is 22.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:29,853 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:29,853 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/C, priority=12, startTime=1733776949327; duration=0sec 2024-12-09T20:42:29,853 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:29,853 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:C 2024-12-09T20:42:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T20:42:29,872 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:29,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:29,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777009871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-09T20:42:29,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:29,874 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T20:42:29,874 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:29,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:29,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777009881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T20:42:30,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T20:42:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777010081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:30,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777010091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:30,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:30,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209593dea31f5494c3ebb639faf5686f63b_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209593dea31f5494c3ebb639faf5686f63b_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:30,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0a92731c42fe4fbb9462838fd5d0fb52, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:30,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0a92731c42fe4fbb9462838fd5d0fb52 is 175, key is test_row_0/A:col10/1733776949653/Put/seqid=0 2024-12-09T20:42:30,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742016_1192 (size=66023) 2024-12-09T20:42:30,122 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0a92731c42fe4fbb9462838fd5d0fb52 2024-12-09T20:42:30,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/b50b2678ac924e6287f2250d86ec0998 is 50, key is test_row_0/B:col10/1733776949653/Put/seqid=0 2024-12-09T20:42:30,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742017_1193 (size=12301) 2024-12-09T20:42:30,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T20:42:30,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:30,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T20:42:30,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T20:42:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777010391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:30,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777010401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:30,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T20:42:30,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:30,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T20:42:30,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/b50b2678ac924e6287f2250d86ec0998 2024-12-09T20:42:30,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/624b713b7def4c08942c572fc1b27d44 is 50, key is test_row_0/C:col10/1733776949653/Put/seqid=0 2024-12-09T20:42:30,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742018_1194 (size=12301) 2024-12-09T20:42:30,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/624b713b7def4c08942c572fc1b27d44 2024-12-09T20:42:30,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/0a92731c42fe4fbb9462838fd5d0fb52 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0a92731c42fe4fbb9462838fd5d0fb52 2024-12-09T20:42:30,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0a92731c42fe4fbb9462838fd5d0fb52, entries=350, sequenceid=391, filesize=64.5 K 2024-12-09T20:42:30,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/b50b2678ac924e6287f2250d86ec0998 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/b50b2678ac924e6287f2250d86ec0998 2024-12-09T20:42:30,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/b50b2678ac924e6287f2250d86ec0998, entries=150, sequenceid=391, filesize=12.0 K 2024-12-09T20:42:30,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/624b713b7def4c08942c572fc1b27d44 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/624b713b7def4c08942c572fc1b27d44 2024-12-09T20:42:30,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/624b713b7def4c08942c572fc1b27d44, entries=150, sequenceid=391, filesize=12.0 K 2024-12-09T20:42:30,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for c7804971d6cc1c0c364cf6c88d0c46fe in 933ms, sequenceid=391, compaction requested=true 2024-12-09T20:42:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:30,603 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:30,603 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:30,604 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:30,604 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/B is initiating minor compaction (all files) 2024-12-09T20:42:30,605 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/B in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,605 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e63d0ac599464ed5aa1fc7df911ac462, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/561690c2ec034f17894421037e693cb5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/b50b2678ac924e6287f2250d86ec0998] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=34.1 K 2024-12-09T20:42:30,605 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:30,605 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/A is initiating minor compaction (all files) 2024-12-09T20:42:30,605 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/A in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,605 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/967d0b45e9504831b3e913156b2cad40, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/e78f4a77b90241a8a14757787daf410e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0a92731c42fe4fbb9462838fd5d0fb52] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=117.5 K 2024-12-09T20:42:30,605 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,605 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/967d0b45e9504831b3e913156b2cad40, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/e78f4a77b90241a8a14757787daf410e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0a92731c42fe4fbb9462838fd5d0fb52] 2024-12-09T20:42:30,606 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting e63d0ac599464ed5aa1fc7df911ac462, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733776947282 2024-12-09T20:42:30,606 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 967d0b45e9504831b3e913156b2cad40, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733776947282 2024-12-09T20:42:30,606 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 561690c2ec034f17894421037e693cb5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733776948450 2024-12-09T20:42:30,606 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e78f4a77b90241a8a14757787daf410e, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733776948450 2024-12-09T20:42:30,607 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a92731c42fe4fbb9462838fd5d0fb52, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733776949623 2024-12-09T20:42:30,607 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b50b2678ac924e6287f2250d86ec0998, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733776949653 2024-12-09T20:42:30,622 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:30,623 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/e9b51729d562430d9c7df9e44bd70cbb is 50, key is test_row_0/B:col10/1733776949653/Put/seqid=0 2024-12-09T20:42:30,637 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:30,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T20:42:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:30,639 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-09T20:42:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:30,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:30,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:30,646 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412091daa38ea10f041d99c95fa4b5b5e50b4_c7804971d6cc1c0c364cf6c88d0c46fe store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:30,648 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412091daa38ea10f041d99c95fa4b5b5e50b4_c7804971d6cc1c0c364cf6c88d0c46fe, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:30,648 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091daa38ea10f041d99c95fa4b5b5e50b4_c7804971d6cc1c0c364cf6c88d0c46fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:30,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742019_1195 (size=12881) 2024-12-09T20:42:30,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742020_1196 (size=4469) 2024-12-09T20:42:30,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097c553ef5b3b94558b48794860996b2ac_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776949757/Put/seqid=0 2024-12-09T20:42:30,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742021_1197 (size=12454) 2024-12-09T20:42:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:30,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777010922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777010922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T20:42:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777011031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777011031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:31,072 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/e9b51729d562430d9c7df9e44bd70cbb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e9b51729d562430d9c7df9e44bd70cbb 2024-12-09T20:42:31,076 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/B of c7804971d6cc1c0c364cf6c88d0c46fe into e9b51729d562430d9c7df9e44bd70cbb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:31,077 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:31,077 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/B, priority=13, startTime=1733776950603; duration=0sec 2024-12-09T20:42:31,077 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:31,077 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:B 2024-12-09T20:42:31,077 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:31,078 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:31,078 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/C is initiating minor compaction (all files) 2024-12-09T20:42:31,078 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/C in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:31,078 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d3a88631cefa483185674067ecf322a7, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/41590ad2d44d4fa382e07f0bb55883a0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/624b713b7def4c08942c572fc1b27d44] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=34.1 K 2024-12-09T20:42:31,079 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d3a88631cefa483185674067ecf322a7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733776947282 2024-12-09T20:42:31,079 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 41590ad2d44d4fa382e07f0bb55883a0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733776948450 2024-12-09T20:42:31,079 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 624b713b7def4c08942c572fc1b27d44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733776949653 2024-12-09T20:42:31,085 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:31,087 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/35c254c5ec784e1c831d322e504f5286 is 50, key is test_row_0/C:col10/1733776949653/Put/seqid=0 2024-12-09T20:42:31,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742022_1198 (size=12881) 2024-12-09T20:42:31,103 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#A#compaction#163 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:31,103 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1b2e5c27a001422f88688349989028f0 is 175, key is test_row_0/A:col10/1733776949653/Put/seqid=0 2024-12-09T20:42:31,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742023_1199 (size=31835) 2024-12-09T20:42:31,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:31,116 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097c553ef5b3b94558b48794860996b2ac_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097c553ef5b3b94558b48794860996b2ac_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/63a977652e944c448e569fdc835c58a3, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/63a977652e944c448e569fdc835c58a3 is 175, key is test_row_0/A:col10/1733776949757/Put/seqid=0 2024-12-09T20:42:31,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742024_1200 (size=31255) 2024-12-09T20:42:31,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777011241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:31,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777011242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:31,496 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/35c254c5ec784e1c831d322e504f5286 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/35c254c5ec784e1c831d322e504f5286 2024-12-09T20:42:31,507 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/C of c7804971d6cc1c0c364cf6c88d0c46fe into 35c254c5ec784e1c831d322e504f5286(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:31,507 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:31,507 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/C, priority=13, startTime=1733776950603; duration=0sec 2024-12-09T20:42:31,507 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:31,507 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:C 2024-12-09T20:42:31,513 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1b2e5c27a001422f88688349989028f0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1b2e5c27a001422f88688349989028f0 2024-12-09T20:42:31,519 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/A of c7804971d6cc1c0c364cf6c88d0c46fe into 1b2e5c27a001422f88688349989028f0(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:31,519 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:31,520 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/A, priority=13, startTime=1733776950603; duration=0sec 2024-12-09T20:42:31,520 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:31,520 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:A 2024-12-09T20:42:31,522 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/63a977652e944c448e569fdc835c58a3 2024-12-09T20:42:31,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f5b89879696b44fc88d6a25868d87e7b is 50, key is test_row_0/B:col10/1733776949757/Put/seqid=0 2024-12-09T20:42:31,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742025_1201 (size=12301) 2024-12-09T20:42:31,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777011551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:31,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:31,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777011551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:31,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:31,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:31,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:31,934 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f5b89879696b44fc88d6a25868d87e7b 2024-12-09T20:42:31,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/7e63ac639ed44966bd1e15cc3328686a is 50, key is test_row_0/C:col10/1733776949757/Put/seqid=0 2024-12-09T20:42:31,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742026_1202 (size=12301) 2024-12-09T20:42:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T20:42:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777012060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777012061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:32,349 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/7e63ac639ed44966bd1e15cc3328686a 2024-12-09T20:42:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/63a977652e944c448e569fdc835c58a3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/63a977652e944c448e569fdc835c58a3 2024-12-09T20:42:32,358 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/63a977652e944c448e569fdc835c58a3, entries=150, sequenceid=428, filesize=30.5 K 2024-12-09T20:42:32,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/f5b89879696b44fc88d6a25868d87e7b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5b89879696b44fc88d6a25868d87e7b 2024-12-09T20:42:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,363 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5b89879696b44fc88d6a25868d87e7b, entries=150, sequenceid=428, filesize=12.0 K 2024-12-09T20:42:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/7e63ac639ed44966bd1e15cc3328686a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/7e63ac639ed44966bd1e15cc3328686a 2024-12-09T20:42:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,371 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/7e63ac639ed44966bd1e15cc3328686a, entries=150, sequenceid=428, filesize=12.0 K 2024-12-09T20:42:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,372 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for c7804971d6cc1c0c364cf6c88d0c46fe in 1733ms, sequenceid=428, compaction requested=false 2024-12-09T20:42:32,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:32,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-09T20:42:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-09T20:42:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-09T20:42:32,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5010 sec 2024-12-09T20:42:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.5050 sec 2024-12-09T20:42:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:33,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:42:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:33,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:33,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:33,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:33,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:33,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f1d535b9d18b4d80a84b90271c388f28_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776950921/Put/seqid=0 2024-12-09T20:42:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742027_1203 (size=12454) 2024-12-09T20:42:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f1d535b9d18b4d80a84b90271c388f28_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f1d535b9d18b4d80a84b90271c388f28_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:33,104 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/c59f6ea0cad94bd49742c2b41e45775e, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:33,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/c59f6ea0cad94bd49742c2b41e45775e is 175, key is test_row_0/A:col10/1733776950921/Put/seqid=0 2024-12-09T20:42:33,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742028_1204 (size=31255) 2024-12-09T20:42:33,122 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=443, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/c59f6ea0cad94bd49742c2b41e45775e 2024-12-09T20:42:33,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/51c0a4b448d64c8eada7bea05b74e36a is 50, key is test_row_0/B:col10/1733776950921/Put/seqid=0 2024-12-09T20:42:33,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777013136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777013138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742029_1205 (size=12301) 2024-12-09T20:42:33,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/51c0a4b448d64c8eada7bea05b74e36a 2024-12-09T20:42:33,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/6b822d63c7394fdfb4bca33cb1b1bb9c is 50, key is test_row_0/C:col10/1733776950921/Put/seqid=0 2024-12-09T20:42:33,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742030_1206 (size=12301) 2024-12-09T20:42:33,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/6b822d63c7394fdfb4bca33cb1b1bb9c 2024-12-09T20:42:33,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/c59f6ea0cad94bd49742c2b41e45775e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/c59f6ea0cad94bd49742c2b41e45775e 2024-12-09T20:42:33,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/c59f6ea0cad94bd49742c2b41e45775e, entries=150, sequenceid=443, filesize=30.5 K 2024-12-09T20:42:33,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/51c0a4b448d64c8eada7bea05b74e36a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/51c0a4b448d64c8eada7bea05b74e36a 2024-12-09T20:42:33,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/51c0a4b448d64c8eada7bea05b74e36a, entries=150, sequenceid=443, filesize=12.0 K 2024-12-09T20:42:33,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/6b822d63c7394fdfb4bca33cb1b1bb9c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6b822d63c7394fdfb4bca33cb1b1bb9c 2024-12-09T20:42:33,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6b822d63c7394fdfb4bca33cb1b1bb9c, entries=150, sequenceid=443, filesize=12.0 K 2024-12-09T20:42:33,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c7804971d6cc1c0c364cf6c88d0c46fe in 137ms, sequenceid=443, compaction requested=true 2024-12-09T20:42:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:33,212 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:33,213 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:33,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:33,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:33,217 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:33,217 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:33,217 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/B is initiating minor compaction (all files) 2024-12-09T20:42:33,217 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/A is initiating minor compaction (all files) 2024-12-09T20:42:33,217 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/A in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:33,217 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/B in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:33,218 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e9b51729d562430d9c7df9e44bd70cbb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5b89879696b44fc88d6a25868d87e7b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/51c0a4b448d64c8eada7bea05b74e36a] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=36.6 K 2024-12-09T20:42:33,218 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1b2e5c27a001422f88688349989028f0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/63a977652e944c448e569fdc835c58a3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/c59f6ea0cad94bd49742c2b41e45775e] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=92.1 K 2024-12-09T20:42:33,218 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:33,218 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1b2e5c27a001422f88688349989028f0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/63a977652e944c448e569fdc835c58a3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/c59f6ea0cad94bd49742c2b41e45775e] 2024-12-09T20:42:33,218 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b51729d562430d9c7df9e44bd70cbb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733776949653 2024-12-09T20:42:33,218 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b2e5c27a001422f88688349989028f0, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733776949653 2024-12-09T20:42:33,219 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f5b89879696b44fc88d6a25868d87e7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733776949757 2024-12-09T20:42:33,220 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a977652e944c448e569fdc835c58a3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733776949757 2024-12-09T20:42:33,220 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c0a4b448d64c8eada7bea05b74e36a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733776950921 2024-12-09T20:42:33,221 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting c59f6ea0cad94bd49742c2b41e45775e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733776950921 2024-12-09T20:42:33,238 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:33,239 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/0d5d6844af244559b628ca93e3f3184e is 50, key is test_row_0/B:col10/1733776950921/Put/seqid=0 2024-12-09T20:42:33,240 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:33,253 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412090eb6450d190d4107a0ce542b578ee55b_c7804971d6cc1c0c364cf6c88d0c46fe store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:33,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T20:42:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:33,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:33,260 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412090eb6450d190d4107a0ce542b578ee55b_c7804971d6cc1c0c364cf6c88d0c46fe, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:33,261 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090eb6450d190d4107a0ce542b578ee55b_c7804971d6cc1c0c364cf6c88d0c46fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:33,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742031_1207 (size=12983) 2024-12-09T20:42:33,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742032_1208 (size=4469) 2024-12-09T20:42:33,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120911145ad7fe4440e09f0a1ebd6d735c3c_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776953133/Put/seqid=0 2024-12-09T20:42:33,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742033_1209 (size=17534) 2024-12-09T20:42:33,273 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#A#compaction#172 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:33,273 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/7c6b2f5a84f84ab1b9c900bb31503ae3 is 175, key is test_row_0/A:col10/1733776950921/Put/seqid=0 2024-12-09T20:42:33,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777013274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777013275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742034_1210 (size=31937) 2024-12-09T20:42:33,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777013381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777013381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777013591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777013591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,670 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/0d5d6844af244559b628ca93e3f3184e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0d5d6844af244559b628ca93e3f3184e 2024-12-09T20:42:33,671 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:33,675 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120911145ad7fe4440e09f0a1ebd6d735c3c_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120911145ad7fe4440e09f0a1ebd6d735c3c_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:33,676 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1aedbb65068d43afa59d8b45c5395d36, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:33,676 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/B of c7804971d6cc1c0c364cf6c88d0c46fe into 0d5d6844af244559b628ca93e3f3184e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:33,676 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:33,676 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/B, priority=13, startTime=1733776953212; duration=0sec 2024-12-09T20:42:33,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1aedbb65068d43afa59d8b45c5395d36 is 175, key is test_row_0/A:col10/1733776953133/Put/seqid=0 2024-12-09T20:42:33,677 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:33,677 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:B 2024-12-09T20:42:33,677 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:33,679 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:33,679 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/C is initiating minor compaction (all files) 2024-12-09T20:42:33,680 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/C in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:33,680 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/35c254c5ec784e1c831d322e504f5286, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/7e63ac639ed44966bd1e15cc3328686a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6b822d63c7394fdfb4bca33cb1b1bb9c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=36.6 K 2024-12-09T20:42:33,680 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 35c254c5ec784e1c831d322e504f5286, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733776949653 2024-12-09T20:42:33,681 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e63ac639ed44966bd1e15cc3328686a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733776949757 2024-12-09T20:42:33,681 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b822d63c7394fdfb4bca33cb1b1bb9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733776950921 2024-12-09T20:42:33,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742035_1211 (size=48639) 2024-12-09T20:42:33,688 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/7c6b2f5a84f84ab1b9c900bb31503ae3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/7c6b2f5a84f84ab1b9c900bb31503ae3 2024-12-09T20:42:33,693 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/A of c7804971d6cc1c0c364cf6c88d0c46fe into 7c6b2f5a84f84ab1b9c900bb31503ae3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:33,693 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:33,693 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/A, priority=13, startTime=1733776953212; duration=0sec 2024-12-09T20:42:33,694 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:33,694 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:33,694 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:A 2024-12-09T20:42:33,694 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/8f605ddc825b4a8493285ef471a4ba7c is 50, key is test_row_0/C:col10/1733776950921/Put/seqid=0 2024-12-09T20:42:33,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742036_1212 (size=12983) 2024-12-09T20:42:33,716 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/8f605ddc825b4a8493285ef471a4ba7c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8f605ddc825b4a8493285ef471a4ba7c 2024-12-09T20:42:33,721 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/C of c7804971d6cc1c0c364cf6c88d0c46fe into 8f605ddc825b4a8493285ef471a4ba7c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:33,721 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:33,721 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/C, priority=13, startTime=1733776953212; duration=0sec 2024-12-09T20:42:33,721 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:33,722 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:C 2024-12-09T20:42:33,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51968 deadline: 1733777013891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,893 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777013901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777013901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:34,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51998 deadline: 1733777014011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:34,011 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in c7804971d6cc1c0c364cf6c88d0c46fe of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18342 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:42:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T20:42:34,041 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:34,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-09T20:42:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T20:42:34,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:34,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:34,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:34,086 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=465, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1aedbb65068d43afa59d8b45c5395d36 2024-12-09T20:42:34,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/64d4cf12c5454b6b8e5f28aec339a0bb is 50, key is test_row_0/B:col10/1733776953133/Put/seqid=0 2024-12-09T20:42:34,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742037_1213 (size=12301) 2024-12-09T20:42:34,113 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T20:42:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T20:42:34,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T20:42:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T20:42:34,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T20:42:34,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777014411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:34,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777014411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:34,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/64d4cf12c5454b6b8e5f28aec339a0bb 2024-12-09T20:42:34,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/237412ad91714ad0b488eb49ded654ab is 50, key is test_row_0/C:col10/1733776953133/Put/seqid=0 2024-12-09T20:42:34,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T20:42:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742038_1214 (size=12301) 2024-12-09T20:42:34,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T20:42:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T20:42:34,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T20:42:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:34,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/237412ad91714ad0b488eb49ded654ab 2024-12-09T20:42:34,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1aedbb65068d43afa59d8b45c5395d36 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1aedbb65068d43afa59d8b45c5395d36 2024-12-09T20:42:34,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1aedbb65068d43afa59d8b45c5395d36, entries=250, sequenceid=465, filesize=47.5 K 2024-12-09T20:42:34,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/64d4cf12c5454b6b8e5f28aec339a0bb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/64d4cf12c5454b6b8e5f28aec339a0bb 2024-12-09T20:42:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/64d4cf12c5454b6b8e5f28aec339a0bb, entries=150, sequenceid=465, filesize=12.0 K 2024-12-09T20:42:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/237412ad91714ad0b488eb49ded654ab as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/237412ad91714ad0b488eb49ded654ab 2024-12-09T20:42:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/237412ad91714ad0b488eb49ded654ab, entries=150, sequenceid=465, filesize=12.0 K 2024-12-09T20:42:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for c7804971d6cc1c0c364cf6c88d0c46fe in 1679ms, sequenceid=465, compaction requested=false 2024-12-09T20:42:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T20:42:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,964 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:42:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c9dcaf05164746afb809bdc36c2a1e9f_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776953272/Put/seqid=0 2024-12-09T20:42:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742039_1215 (size=9914) 2024-12-09T20:42:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T20:42:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,384 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c9dcaf05164746afb809bdc36c2a1e9f_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c9dcaf05164746afb809bdc36c2a1e9f_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/4f8646e6066e497ebaf52ac57bd4322c, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/4f8646e6066e497ebaf52ac57bd4322c is 175, key is test_row_0/A:col10/1733776953272/Put/seqid=0 2024-12-09T20:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742040_1216 (size=22561) 2024-12-09T20:42:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:35,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. as already flushing 2024-12-09T20:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:35,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733776955485, max timestamp=1733776955500Num uniques 859; 2024-12-09T20:42:35,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:42:35,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733776955485, max timestamp=1733776955500Num uniques 859; 2024-12-09T20:42:35,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:42:35,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733776955485, max timestamp=1733776955500Num uniques 859; 2024-12-09T20:42:35,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:42:35,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777015546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:35,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777015547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:35,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777015661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:35,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777015661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:35,680 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-12-09T20:42:35,681 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:35,681 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:35,681 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:35,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:35,682 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-12-09T20:42:35,682 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:35,682 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:35,682 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:35,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:35,688 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-12-09T20:42:35,688 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:35,688 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:35,688 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:35,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:35,693 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-12-09T20:42:35,693 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:35,693 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:35,693 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:35,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:35,797 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=478, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/4f8646e6066e497ebaf52ac57bd4322c 2024-12-09T20:42:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/d58d3a78ac55403082e498ac554a30be is 50, key is test_row_0/B:col10/1733776953272/Put/seqid=0 2024-12-09T20:42:35,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742041_1217 (size=9857) 2024-12-09T20:42:35,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:35,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777015872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777015871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:36,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:36,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51954 deadline: 1733777016181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52006 deadline: 1733777016182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:42:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T20:42:36,211 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/d58d3a78ac55403082e498ac554a30be 2024-12-09T20:42:36,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/a6df1951004b48c1b48312f6e1709f96 is 50, key is test_row_0/C:col10/1733776953272/Put/seqid=0 2024-12-09T20:42:36,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742042_1218 (size=9857) 2024-12-09T20:42:36,630 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/a6df1951004b48c1b48312f6e1709f96 2024-12-09T20:42:36,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/4f8646e6066e497ebaf52ac57bd4322c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4f8646e6066e497ebaf52ac57bd4322c 2024-12-09T20:42:36,644 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4f8646e6066e497ebaf52ac57bd4322c, entries=100, sequenceid=478, filesize=22.0 K 2024-12-09T20:42:36,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/d58d3a78ac55403082e498ac554a30be as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/d58d3a78ac55403082e498ac554a30be 2024-12-09T20:42:36,650 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/d58d3a78ac55403082e498ac554a30be, entries=100, sequenceid=478, filesize=9.6 K 2024-12-09T20:42:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/a6df1951004b48c1b48312f6e1709f96 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/a6df1951004b48c1b48312f6e1709f96 2024-12-09T20:42:36,655 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/a6df1951004b48c1b48312f6e1709f96, entries=100, sequenceid=478, filesize=9.6 K 2024-12-09T20:42:36,656 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c7804971d6cc1c0c364cf6c88d0c46fe in 1693ms, sequenceid=478, compaction requested=true 2024-12-09T20:42:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-09T20:42:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-09T20:42:36,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-09T20:42:36,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6140 sec 2024-12-09T20:42:36,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.6180 sec 2024-12-09T20:42:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:36,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-09T20:42:36,693 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-12-09T20:42:36,693 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-12-09T20:42:36,694 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:36,694 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:36,694 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:36,694 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:36,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:36,694 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:36,694 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:36,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:42:36,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:36,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:36,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:42:36,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:36,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:36,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:42:36,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f098817bc2fe4169bdf743095a3e11c3_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776955544/Put/seqid=0 2024-12-09T20:42:36,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742043_1219 (size=12454) 2024-12-09T20:42:37,107 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:37,111 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f098817bc2fe4169bdf743095a3e11c3_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f098817bc2fe4169bdf743095a3e11c3_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:37,112 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1db08b0122b5491d88ea9fce53a66c58, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:37,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1db08b0122b5491d88ea9fce53a66c58 is 175, key is test_row_0/A:col10/1733776955544/Put/seqid=0 2024-12-09T20:42:37,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742044_1220 (size=31255) 2024-12-09T20:42:37,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=516, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1db08b0122b5491d88ea9fce53a66c58 2024-12-09T20:42:37,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/88356c00fc71458fbd55b156b7a37100 is 50, key is test_row_0/B:col10/1733776955544/Put/seqid=0 2024-12-09T20:42:37,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742045_1221 (size=12301) 2024-12-09T20:42:37,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/88356c00fc71458fbd55b156b7a37100 2024-12-09T20:42:37,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/b16127a875154ab38841efe94b3ec43b is 50, key is test_row_0/C:col10/1733776955544/Put/seqid=0 2024-12-09T20:42:37,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742046_1222 (size=12301) 2024-12-09T20:42:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T20:42:38,211 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:38,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/b16127a875154ab38841efe94b3ec43b 2024-12-09T20:42:38,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/1db08b0122b5491d88ea9fce53a66c58 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1db08b0122b5491d88ea9fce53a66c58 2024-12-09T20:42:38,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1db08b0122b5491d88ea9fce53a66c58, entries=150, sequenceid=516, filesize=30.5 K 2024-12-09T20:42:38,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/88356c00fc71458fbd55b156b7a37100 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/88356c00fc71458fbd55b156b7a37100 2024-12-09T20:42:38,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/88356c00fc71458fbd55b156b7a37100, entries=150, sequenceid=516, filesize=12.0 K 2024-12-09T20:42:38,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/b16127a875154ab38841efe94b3ec43b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/b16127a875154ab38841efe94b3ec43b 2024-12-09T20:42:38,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/b16127a875154ab38841efe94b3ec43b, entries=150, sequenceid=516, filesize=12.0 K 2024-12-09T20:42:38,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=0 B/0 for c7804971d6cc1c0c364cf6c88d0c46fe in 1672ms, sequenceid=516, compaction requested=true 2024-12-09T20:42:38,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:38,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:38,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:38,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:38,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:38,366 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:38,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7804971d6cc1c0c364cf6c88d0c46fe:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:38,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:38,366 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:38,367 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:38,367 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:38,367 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/B is initiating minor compaction (all files) 2024-12-09T20:42:38,367 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/A is initiating minor compaction (all files) 2024-12-09T20:42:38,367 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/B in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:38,367 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/A in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:38,367 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0d5d6844af244559b628ca93e3f3184e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/64d4cf12c5454b6b8e5f28aec339a0bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/d58d3a78ac55403082e498ac554a30be, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/88356c00fc71458fbd55b156b7a37100] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=46.3 K 2024-12-09T20:42:38,367 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/7c6b2f5a84f84ab1b9c900bb31503ae3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1aedbb65068d43afa59d8b45c5395d36, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4f8646e6066e497ebaf52ac57bd4322c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1db08b0122b5491d88ea9fce53a66c58] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=131.2 K 2024-12-09T20:42:38,367 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:38,367 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/7c6b2f5a84f84ab1b9c900bb31503ae3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1aedbb65068d43afa59d8b45c5395d36, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4f8646e6066e497ebaf52ac57bd4322c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1db08b0122b5491d88ea9fce53a66c58] 2024-12-09T20:42:38,367 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d5d6844af244559b628ca93e3f3184e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733776950921 2024-12-09T20:42:38,367 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6b2f5a84f84ab1b9c900bb31503ae3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733776950921 2024-12-09T20:42:38,367 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 64d4cf12c5454b6b8e5f28aec339a0bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1733776953129 2024-12-09T20:42:38,368 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aedbb65068d43afa59d8b45c5395d36, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1733776953129 2024-12-09T20:42:38,368 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d58d3a78ac55403082e498ac554a30be, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733776953272 2024-12-09T20:42:38,368 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 88356c00fc71458fbd55b156b7a37100, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733776955544 2024-12-09T20:42:38,368 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f8646e6066e497ebaf52ac57bd4322c, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733776953272 2024-12-09T20:42:38,368 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1db08b0122b5491d88ea9fce53a66c58, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733776955544 2024-12-09T20:42:38,379 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#B#compaction#183 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:38,380 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/977007f7cd4f48a8809e3a6a39a2e4ca is 50, key is test_row_0/B:col10/1733776955544/Put/seqid=0 2024-12-09T20:42:38,381 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:38,383 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209330d50fd87df4883b48af5c96bffd52e_c7804971d6cc1c0c364cf6c88d0c46fe store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:38,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742047_1223 (size=13119) 2024-12-09T20:42:38,405 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209330d50fd87df4883b48af5c96bffd52e_c7804971d6cc1c0c364cf6c88d0c46fe, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:38,405 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209330d50fd87df4883b48af5c96bffd52e_c7804971d6cc1c0c364cf6c88d0c46fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:38,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742048_1224 (size=4469) 2024-12-09T20:42:38,415 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#A#compaction#184 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:38,416 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ec4b5adc78514398b4b1111b23b2b090 is 175, key is test_row_0/A:col10/1733776955544/Put/seqid=0 2024-12-09T20:42:38,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742049_1225 (size=32073) 2024-12-09T20:42:38,438 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ec4b5adc78514398b4b1111b23b2b090 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ec4b5adc78514398b4b1111b23b2b090 2024-12-09T20:42:38,445 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/A of c7804971d6cc1c0c364cf6c88d0c46fe into ec4b5adc78514398b4b1111b23b2b090(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:38,445 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:38,445 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/A, priority=12, startTime=1733776958366; duration=0sec 2024-12-09T20:42:38,445 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:38,445 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:A 2024-12-09T20:42:38,445 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:38,447 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:38,447 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): c7804971d6cc1c0c364cf6c88d0c46fe/C is initiating minor compaction (all files) 2024-12-09T20:42:38,447 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7804971d6cc1c0c364cf6c88d0c46fe/C in TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:38,447 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8f605ddc825b4a8493285ef471a4ba7c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/237412ad91714ad0b488eb49ded654ab, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/a6df1951004b48c1b48312f6e1709f96, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/b16127a875154ab38841efe94b3ec43b] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp, totalSize=46.3 K 2024-12-09T20:42:38,448 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f605ddc825b4a8493285ef471a4ba7c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733776950921 2024-12-09T20:42:38,448 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 237412ad91714ad0b488eb49ded654ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1733776953129 2024-12-09T20:42:38,449 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6df1951004b48c1b48312f6e1709f96, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733776953272 2024-12-09T20:42:38,449 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting b16127a875154ab38841efe94b3ec43b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733776955544 2024-12-09T20:42:38,461 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7804971d6cc1c0c364cf6c88d0c46fe#C#compaction#185 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:38,462 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/65bfb4fd9794449388668fa9747a860c is 50, key is test_row_0/C:col10/1733776955544/Put/seqid=0 2024-12-09T20:42:38,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742050_1226 (size=13119) 2024-12-09T20:42:38,795 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/977007f7cd4f48a8809e3a6a39a2e4ca as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/977007f7cd4f48a8809e3a6a39a2e4ca 2024-12-09T20:42:38,801 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/B of c7804971d6cc1c0c364cf6c88d0c46fe into 977007f7cd4f48a8809e3a6a39a2e4ca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:38,801 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:38,801 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/B, priority=12, startTime=1733776958366; duration=0sec 2024-12-09T20:42:38,801 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:38,801 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:B 2024-12-09T20:42:38,878 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/65bfb4fd9794449388668fa9747a860c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/65bfb4fd9794449388668fa9747a860c 2024-12-09T20:42:38,883 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7804971d6cc1c0c364cf6c88d0c46fe/C of c7804971d6cc1c0c364cf6c88d0c46fe into 65bfb4fd9794449388668fa9747a860c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:38,884 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7804971d6cc1c0c364cf6c88d0c46fe: 2024-12-09T20:42:38,884 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., storeName=c7804971d6cc1c0c364cf6c88d0c46fe/C, priority=12, startTime=1733776958366; duration=0sec 2024-12-09T20:42:38,884 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:38,884 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7804971d6cc1c0c364cf6c88d0c46fe:C 2024-12-09T20:42:39,353 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-12-09T20:42:39,353 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:39,353 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:39,353 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:39,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:43,975 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-12-09T20:42:43,975 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:43,975 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:43,976 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:43,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:44,096 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-12-09T20:42:44,096 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:42:44,096 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:44,097 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:44,097 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:44,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T20:42:44,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 3 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 145 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 218 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3786 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3910 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1616 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4848 rows 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1622 2024-12-09T20:42:44,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4866 rows 2024-12-09T20:42:44,098 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:42:44,098 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:42:44,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:44,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:44,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:42:44,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T20:42:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T20:42:44,106 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776964105"}]},"ts":"1733776964105"} 2024-12-09T20:42:44,108 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T20:42:44,108 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T20:42:44,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T20:42:44,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, UNASSIGN}] 2024-12-09T20:42:44,112 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, UNASSIGN 2024-12-09T20:42:44,113 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=c7804971d6cc1c0c364cf6c88d0c46fe, regionState=CLOSING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:44,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, UNASSIGN because future has completed 2024-12-09T20:42:44,115 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T20:42:44,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:42:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T20:42:44,267 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:44,267 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing c7804971d6cc1c0c364cf6c88d0c46fe, disabling compactions & flushes 2024-12-09T20:42:44,268 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. after waiting 0 ms 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:44,268 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing c7804971d6cc1c0c364cf6c88d0c46fe 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=A 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=B 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7804971d6cc1c0c364cf6c88d0c46fe, store=C 2024-12-09T20:42:44,268 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:44,290 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098ad4aeee29234666a8da9a5dac013c33_c7804971d6cc1c0c364cf6c88d0c46fe is 50, key is test_row_0/A:col10/1733776963972/Put/seqid=0 2024-12-09T20:42:44,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742051_1227 (size=12454) 2024-12-09T20:42:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T20:42:44,696 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:44,701 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098ad4aeee29234666a8da9a5dac013c33_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098ad4aeee29234666a8da9a5dac013c33_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:44,702 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ead8f429d9a545e5ab1863a4f25956cc, store: [table=TestAcidGuarantees family=A region=c7804971d6cc1c0c364cf6c88d0c46fe] 2024-12-09T20:42:44,703 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ead8f429d9a545e5ab1863a4f25956cc is 175, key is test_row_0/A:col10/1733776963972/Put/seqid=0 2024-12-09T20:42:44,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742052_1228 (size=31255) 2024-12-09T20:42:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T20:42:45,107 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=525, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ead8f429d9a545e5ab1863a4f25956cc 2024-12-09T20:42:45,113 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/9036f5dec5a74d6ab9b2eb6f70e6b32e is 50, key is test_row_0/B:col10/1733776963972/Put/seqid=0 2024-12-09T20:42:45,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742053_1229 (size=12301) 2024-12-09T20:42:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T20:42:45,518 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/9036f5dec5a74d6ab9b2eb6f70e6b32e 2024-12-09T20:42:45,524 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/1747ea45758747b09d6ce90e234e06ff is 50, key is test_row_0/C:col10/1733776963972/Put/seqid=0 2024-12-09T20:42:45,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742054_1230 (size=12301) 2024-12-09T20:42:45,932 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/1747ea45758747b09d6ce90e234e06ff 2024-12-09T20:42:45,937 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/A/ead8f429d9a545e5ab1863a4f25956cc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ead8f429d9a545e5ab1863a4f25956cc 2024-12-09T20:42:45,941 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ead8f429d9a545e5ab1863a4f25956cc, entries=150, sequenceid=525, filesize=30.5 K 2024-12-09T20:42:45,942 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/B/9036f5dec5a74d6ab9b2eb6f70e6b32e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/9036f5dec5a74d6ab9b2eb6f70e6b32e 2024-12-09T20:42:45,946 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/9036f5dec5a74d6ab9b2eb6f70e6b32e, entries=150, sequenceid=525, filesize=12.0 K 2024-12-09T20:42:45,947 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/.tmp/C/1747ea45758747b09d6ce90e234e06ff as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/1747ea45758747b09d6ce90e234e06ff 2024-12-09T20:42:45,951 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/1747ea45758747b09d6ce90e234e06ff, entries=150, sequenceid=525, filesize=12.0 K 2024-12-09T20:42:45,952 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c7804971d6cc1c0c364cf6c88d0c46fe in 1684ms, sequenceid=525, compaction requested=false 2024-12-09T20:42:45,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d418365a90de4a94b840cc73c49ef42e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/fb95bc54a72c47f1b7770a26e3a6b798, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/f848ab7d22ab46f19156c8f65dea9b0e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/41a173f92ec741d29f83f50a586bda93, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d65e4c37789f471fa3caa480d33d8407, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ba6631af64834353b018957ac783a73c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/9fe589408b1b422d9f5f6b897e86220f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4db8ae316ad240e7a77676d97d7dd210, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0dd099144b5c429eb8e7108f5ac82046, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/19c51d42fbe44bff93a770437b0e904e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0951dfade88a4ae59481bbabc4b08fa1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/91ffa3c840bf4dba84039ebb6c49e2b4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ed60108a59f4480ea1167deed583428e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/b97b894e263e4a48a53c9779afceadc6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/967d0b45e9504831b3e913156b2cad40, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/e78f4a77b90241a8a14757787daf410e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0a92731c42fe4fbb9462838fd5d0fb52, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1b2e5c27a001422f88688349989028f0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/63a977652e944c448e569fdc835c58a3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/7c6b2f5a84f84ab1b9c900bb31503ae3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/c59f6ea0cad94bd49742c2b41e45775e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1aedbb65068d43afa59d8b45c5395d36, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4f8646e6066e497ebaf52ac57bd4322c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1db08b0122b5491d88ea9fce53a66c58] to archive 2024-12-09T20:42:45,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:42:45,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d418365a90de4a94b840cc73c49ef42e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d418365a90de4a94b840cc73c49ef42e 2024-12-09T20:42:45,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/fb95bc54a72c47f1b7770a26e3a6b798 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/fb95bc54a72c47f1b7770a26e3a6b798 2024-12-09T20:42:45,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/f848ab7d22ab46f19156c8f65dea9b0e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/f848ab7d22ab46f19156c8f65dea9b0e 2024-12-09T20:42:45,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/41a173f92ec741d29f83f50a586bda93 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/41a173f92ec741d29f83f50a586bda93 2024-12-09T20:42:45,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d65e4c37789f471fa3caa480d33d8407 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/d65e4c37789f471fa3caa480d33d8407 2024-12-09T20:42:45,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ba6631af64834353b018957ac783a73c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ba6631af64834353b018957ac783a73c 2024-12-09T20:42:45,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/9fe589408b1b422d9f5f6b897e86220f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/9fe589408b1b422d9f5f6b897e86220f 2024-12-09T20:42:45,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4db8ae316ad240e7a77676d97d7dd210 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4db8ae316ad240e7a77676d97d7dd210 2024-12-09T20:42:45,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0dd099144b5c429eb8e7108f5ac82046 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0dd099144b5c429eb8e7108f5ac82046 2024-12-09T20:42:45,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/19c51d42fbe44bff93a770437b0e904e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/19c51d42fbe44bff93a770437b0e904e 2024-12-09T20:42:45,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0951dfade88a4ae59481bbabc4b08fa1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0951dfade88a4ae59481bbabc4b08fa1 2024-12-09T20:42:45,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/91ffa3c840bf4dba84039ebb6c49e2b4 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/91ffa3c840bf4dba84039ebb6c49e2b4 2024-12-09T20:42:45,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ed60108a59f4480ea1167deed583428e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ed60108a59f4480ea1167deed583428e 2024-12-09T20:42:45,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/b97b894e263e4a48a53c9779afceadc6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/b97b894e263e4a48a53c9779afceadc6 2024-12-09T20:42:45,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/967d0b45e9504831b3e913156b2cad40 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/967d0b45e9504831b3e913156b2cad40 2024-12-09T20:42:45,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/e78f4a77b90241a8a14757787daf410e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/e78f4a77b90241a8a14757787daf410e 2024-12-09T20:42:45,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0a92731c42fe4fbb9462838fd5d0fb52 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/0a92731c42fe4fbb9462838fd5d0fb52 2024-12-09T20:42:45,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1b2e5c27a001422f88688349989028f0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1b2e5c27a001422f88688349989028f0 2024-12-09T20:42:45,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/63a977652e944c448e569fdc835c58a3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/63a977652e944c448e569fdc835c58a3 2024-12-09T20:42:45,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/7c6b2f5a84f84ab1b9c900bb31503ae3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/7c6b2f5a84f84ab1b9c900bb31503ae3 2024-12-09T20:42:45,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/c59f6ea0cad94bd49742c2b41e45775e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/c59f6ea0cad94bd49742c2b41e45775e 2024-12-09T20:42:45,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1aedbb65068d43afa59d8b45c5395d36 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1aedbb65068d43afa59d8b45c5395d36 2024-12-09T20:42:45,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4f8646e6066e497ebaf52ac57bd4322c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/4f8646e6066e497ebaf52ac57bd4322c 2024-12-09T20:42:45,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1db08b0122b5491d88ea9fce53a66c58 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/1db08b0122b5491d88ea9fce53a66c58 2024-12-09T20:42:45,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0453aabb3ab4466bafb6834575d65dd2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/da57747d85df470e818c4777cd69ae35, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/c1e745226a7941558f69c3ad1f6f72ca, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ee7b00b3551146ef8d5916896a86f14a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/bcc67bb4ef454d0486c1906ebce2e42f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/7718d2f826c9494e8a7f9a0f30837fc3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f30ca23d42a8429089ca5ab48929b39c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5bb4aa15acb4da8b0b5817a3c657e73, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/63f999f652d246ad82fd106a166afac2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e71d01720162470396ee363546540858, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ec0af045ade44747a7702c27ec9b437d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/3e6aaa17a62d41dba23d12fc25491d92, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/03363229df004a3e9c5fd9f194e1d06b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e63d0ac599464ed5aa1fc7df911ac462, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f87035f9017b431ea2e7a1f46a6b09ef, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/561690c2ec034f17894421037e693cb5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e9b51729d562430d9c7df9e44bd70cbb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/b50b2678ac924e6287f2250d86ec0998, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5b89879696b44fc88d6a25868d87e7b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0d5d6844af244559b628ca93e3f3184e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/51c0a4b448d64c8eada7bea05b74e36a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/64d4cf12c5454b6b8e5f28aec339a0bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/d58d3a78ac55403082e498ac554a30be, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/88356c00fc71458fbd55b156b7a37100] to archive 2024-12-09T20:42:45,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:42:45,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0453aabb3ab4466bafb6834575d65dd2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0453aabb3ab4466bafb6834575d65dd2 2024-12-09T20:42:45,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/da57747d85df470e818c4777cd69ae35 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/da57747d85df470e818c4777cd69ae35 2024-12-09T20:42:45,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/c1e745226a7941558f69c3ad1f6f72ca to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/c1e745226a7941558f69c3ad1f6f72ca 2024-12-09T20:42:45,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ee7b00b3551146ef8d5916896a86f14a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ee7b00b3551146ef8d5916896a86f14a 2024-12-09T20:42:45,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/bcc67bb4ef454d0486c1906ebce2e42f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/bcc67bb4ef454d0486c1906ebce2e42f 2024-12-09T20:42:45,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/7718d2f826c9494e8a7f9a0f30837fc3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/7718d2f826c9494e8a7f9a0f30837fc3 2024-12-09T20:42:45,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f30ca23d42a8429089ca5ab48929b39c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f30ca23d42a8429089ca5ab48929b39c 2024-12-09T20:42:45,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5bb4aa15acb4da8b0b5817a3c657e73 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5bb4aa15acb4da8b0b5817a3c657e73 2024-12-09T20:42:45,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/63f999f652d246ad82fd106a166afac2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/63f999f652d246ad82fd106a166afac2 2024-12-09T20:42:45,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e71d01720162470396ee363546540858 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e71d01720162470396ee363546540858 2024-12-09T20:42:45,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ec0af045ade44747a7702c27ec9b437d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/ec0af045ade44747a7702c27ec9b437d 2024-12-09T20:42:45,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/3e6aaa17a62d41dba23d12fc25491d92 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/3e6aaa17a62d41dba23d12fc25491d92 2024-12-09T20:42:45,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/03363229df004a3e9c5fd9f194e1d06b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/03363229df004a3e9c5fd9f194e1d06b 2024-12-09T20:42:45,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e63d0ac599464ed5aa1fc7df911ac462 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e63d0ac599464ed5aa1fc7df911ac462 2024-12-09T20:42:45,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f87035f9017b431ea2e7a1f46a6b09ef to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f87035f9017b431ea2e7a1f46a6b09ef 2024-12-09T20:42:45,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/561690c2ec034f17894421037e693cb5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/561690c2ec034f17894421037e693cb5 2024-12-09T20:42:45,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e9b51729d562430d9c7df9e44bd70cbb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/e9b51729d562430d9c7df9e44bd70cbb 2024-12-09T20:42:46,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/b50b2678ac924e6287f2250d86ec0998 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/b50b2678ac924e6287f2250d86ec0998 2024-12-09T20:42:46,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5b89879696b44fc88d6a25868d87e7b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/f5b89879696b44fc88d6a25868d87e7b 2024-12-09T20:42:46,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0d5d6844af244559b628ca93e3f3184e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/0d5d6844af244559b628ca93e3f3184e 2024-12-09T20:42:46,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/51c0a4b448d64c8eada7bea05b74e36a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/51c0a4b448d64c8eada7bea05b74e36a 2024-12-09T20:42:46,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/64d4cf12c5454b6b8e5f28aec339a0bb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/64d4cf12c5454b6b8e5f28aec339a0bb 2024-12-09T20:42:46,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/d58d3a78ac55403082e498ac554a30be to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/d58d3a78ac55403082e498ac554a30be 2024-12-09T20:42:46,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/88356c00fc71458fbd55b156b7a37100 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/88356c00fc71458fbd55b156b7a37100 2024-12-09T20:42:46,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/687f51b6dd124ecbb2641730c8db765e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/e8b7ef1013f54289aeb86e8192a1b137, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8b10349c36cb4b84a6f5f58fb5bb9197, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f478e49650774032af2253d734de1203, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f93571d5f3544df58b3f886f59fc461c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/5833b527770a4fdab24a84210f954c3a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d0ebfc0e2984403ebd41bac3d588e4c9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/c29337ff08594d4fbe08afb4dcd585ee, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/cf5486ed07344fa5b7930822724ff8e4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6c67e60f5821418e9dca30193079c228, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/2092b2b84f6a423181933a2a539b587a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d6f662914ab647879257a83bb39587f2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/9a3ee2c8bee448e3b7c47406f6a94dbf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d3a88631cefa483185674067ecf322a7, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/19143828d74a4a42ac8781b99486a3fe, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/41590ad2d44d4fa382e07f0bb55883a0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/35c254c5ec784e1c831d322e504f5286, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/624b713b7def4c08942c572fc1b27d44, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/7e63ac639ed44966bd1e15cc3328686a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8f605ddc825b4a8493285ef471a4ba7c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6b822d63c7394fdfb4bca33cb1b1bb9c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/237412ad91714ad0b488eb49ded654ab, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/a6df1951004b48c1b48312f6e1709f96, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/b16127a875154ab38841efe94b3ec43b] to archive 2024-12-09T20:42:46,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:42:46,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/687f51b6dd124ecbb2641730c8db765e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/687f51b6dd124ecbb2641730c8db765e 2024-12-09T20:42:46,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/e8b7ef1013f54289aeb86e8192a1b137 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/e8b7ef1013f54289aeb86e8192a1b137 2024-12-09T20:42:46,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8b10349c36cb4b84a6f5f58fb5bb9197 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8b10349c36cb4b84a6f5f58fb5bb9197 2024-12-09T20:42:46,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f478e49650774032af2253d734de1203 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f478e49650774032af2253d734de1203 2024-12-09T20:42:46,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f93571d5f3544df58b3f886f59fc461c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/f93571d5f3544df58b3f886f59fc461c 2024-12-09T20:42:46,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/5833b527770a4fdab24a84210f954c3a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/5833b527770a4fdab24a84210f954c3a 2024-12-09T20:42:46,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d0ebfc0e2984403ebd41bac3d588e4c9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d0ebfc0e2984403ebd41bac3d588e4c9 2024-12-09T20:42:46,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/c29337ff08594d4fbe08afb4dcd585ee to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/c29337ff08594d4fbe08afb4dcd585ee 2024-12-09T20:42:46,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/cf5486ed07344fa5b7930822724ff8e4 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/cf5486ed07344fa5b7930822724ff8e4 2024-12-09T20:42:46,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6c67e60f5821418e9dca30193079c228 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6c67e60f5821418e9dca30193079c228 2024-12-09T20:42:46,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/2092b2b84f6a423181933a2a539b587a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/2092b2b84f6a423181933a2a539b587a 2024-12-09T20:42:46,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d6f662914ab647879257a83bb39587f2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d6f662914ab647879257a83bb39587f2 2024-12-09T20:42:46,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/9a3ee2c8bee448e3b7c47406f6a94dbf to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/9a3ee2c8bee448e3b7c47406f6a94dbf 2024-12-09T20:42:46,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d3a88631cefa483185674067ecf322a7 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/d3a88631cefa483185674067ecf322a7 2024-12-09T20:42:46,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/19143828d74a4a42ac8781b99486a3fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/19143828d74a4a42ac8781b99486a3fe 2024-12-09T20:42:46,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/41590ad2d44d4fa382e07f0bb55883a0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/41590ad2d44d4fa382e07f0bb55883a0 2024-12-09T20:42:46,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/35c254c5ec784e1c831d322e504f5286 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/35c254c5ec784e1c831d322e504f5286 2024-12-09T20:42:46,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/624b713b7def4c08942c572fc1b27d44 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/624b713b7def4c08942c572fc1b27d44 2024-12-09T20:42:46,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/7e63ac639ed44966bd1e15cc3328686a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/7e63ac639ed44966bd1e15cc3328686a 2024-12-09T20:42:46,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8f605ddc825b4a8493285ef471a4ba7c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/8f605ddc825b4a8493285ef471a4ba7c 2024-12-09T20:42:46,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6b822d63c7394fdfb4bca33cb1b1bb9c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/6b822d63c7394fdfb4bca33cb1b1bb9c 2024-12-09T20:42:46,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/237412ad91714ad0b488eb49ded654ab to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/237412ad91714ad0b488eb49ded654ab 2024-12-09T20:42:46,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/a6df1951004b48c1b48312f6e1709f96 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/a6df1951004b48c1b48312f6e1709f96 2024-12-09T20:42:46,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/b16127a875154ab38841efe94b3ec43b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/b16127a875154ab38841efe94b3ec43b 2024-12-09T20:42:46,036 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/recovered.edits/528.seqid, newMaxSeqId=528, maxSeqId=4 2024-12-09T20:42:46,037 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. 2024-12-09T20:42:46,037 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for c7804971d6cc1c0c364cf6c88d0c46fe: Waiting for close lock at 1733776964268Running coprocessor pre-close hooks at 1733776964268Disabling compacts and flushes for region at 1733776964268Disabling writes for close at 1733776964268Obtaining lock to block concurrent updates at 1733776964268Preparing flush snapshotting stores in c7804971d6cc1c0c364cf6c88d0c46fe at 1733776964268Finished memstore snapshotting TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733776964268Flushing stores of TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe. at 1733776964269 (+1 ms)Flushing c7804971d6cc1c0c364cf6c88d0c46fe/A: creating writer at 1733776964269Flushing mob file c7804971d6cc1c0c364cf6c88d0c46fe/A: appending metadata at 1733776964290 (+21 ms)Flushing mob file c7804971d6cc1c0c364cf6c88d0c46fe/A: closing flushed file at 1733776964290Flushing c7804971d6cc1c0c364cf6c88d0c46fe/A: appending metadata at 1733776964702 (+412 ms)Flushing c7804971d6cc1c0c364cf6c88d0c46fe/A: closing flushed file at 1733776964702Flushing c7804971d6cc1c0c364cf6c88d0c46fe/B: creating writer at 1733776965111 (+409 ms)Flushing c7804971d6cc1c0c364cf6c88d0c46fe/B: appending metadata at 1733776965113 (+2 ms)Flushing c7804971d6cc1c0c364cf6c88d0c46fe/B: closing flushed file at 1733776965113Flushing c7804971d6cc1c0c364cf6c88d0c46fe/C: creating writer at 1733776965521 (+408 ms)Flushing c7804971d6cc1c0c364cf6c88d0c46fe/C: appending metadata at 1733776965524 (+3 ms)Flushing c7804971d6cc1c0c364cf6c88d0c46fe/C: closing flushed file at 1733776965524Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b721f93: reopening flushed file at 1733776965936 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a0bb736: reopening flushed file at 1733776965941 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b840692: reopening flushed file at 1733776965946 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c7804971d6cc1c0c364cf6c88d0c46fe in 1684ms, sequenceid=525, compaction requested=false at 1733776965952 (+6 ms)Writing region close event to WAL at 1733776966033 (+81 ms)Running coprocessor post-close hooks at 1733776966037 (+4 ms)Closed at 1733776966037 2024-12-09T20:42:46,039 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,039 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=c7804971d6cc1c0c364cf6c88d0c46fe, regionState=CLOSED 2024-12-09T20:42:46,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:42:46,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-09T20:42:46,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure c7804971d6cc1c0c364cf6c88d0c46fe, server=00c23a7251f8,36519,1733776895755 in 1.9260 sec 2024-12-09T20:42:46,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-09T20:42:46,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7804971d6cc1c0c364cf6c88d0c46fe, UNASSIGN in 1.9320 sec 2024-12-09T20:42:46,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-09T20:42:46,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9360 sec 2024-12-09T20:42:46,047 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776966047"}]},"ts":"1733776966047"} 2024-12-09T20:42:46,049 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T20:42:46,049 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T20:42:46,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9490 sec 2024-12-09T20:42:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T20:42:46,252 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:46,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T20:42:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:46,254 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T20:42:46,255 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:46,258 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,261 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/recovered.edits] 2024-12-09T20:42:46,263 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ead8f429d9a545e5ab1863a4f25956cc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ead8f429d9a545e5ab1863a4f25956cc 2024-12-09T20:42:46,264 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ec4b5adc78514398b4b1111b23b2b090 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/A/ec4b5adc78514398b4b1111b23b2b090 2024-12-09T20:42:46,266 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/9036f5dec5a74d6ab9b2eb6f70e6b32e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/9036f5dec5a74d6ab9b2eb6f70e6b32e 2024-12-09T20:42:46,267 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/977007f7cd4f48a8809e3a6a39a2e4ca to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/B/977007f7cd4f48a8809e3a6a39a2e4ca 2024-12-09T20:42:46,269 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/1747ea45758747b09d6ce90e234e06ff to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/1747ea45758747b09d6ce90e234e06ff 2024-12-09T20:42:46,270 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/65bfb4fd9794449388668fa9747a860c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/C/65bfb4fd9794449388668fa9747a860c 2024-12-09T20:42:46,272 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/recovered.edits/528.seqid to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe/recovered.edits/528.seqid 2024-12-09T20:42:46,272 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,272 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T20:42:46,273 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T20:42:46,273 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T20:42:46,276 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209059a6195c4a5429c8c8f79e1d09be8c0_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209059a6195c4a5429c8c8f79e1d09be8c0_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,278 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120911145ad7fe4440e09f0a1ebd6d735c3c_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120911145ad7fe4440e09f0a1ebd6d735c3c_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,279 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120915b8ccad69604b66b240a86406cdeead_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120915b8ccad69604b66b240a86406cdeead_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,281 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092a85ee51ae084b3780600ef46be2208e_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092a85ee51ae084b3780600ef46be2208e_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,282 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209332e84e0d1434bdbb10278ef70e1f718_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209332e84e0d1434bdbb10278ef70e1f718_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,283 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093761585be0204aa1b154880cc68d8521_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093761585be0204aa1b154880cc68d8521_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,285 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209393f3b427ce541b291f288dca7e5f572_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209393f3b427ce541b291f288dca7e5f572_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,286 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120951f17abf39f541209ca3119545a6a90a_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120951f17abf39f541209ca3119545a6a90a_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,287 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209593dea31f5494c3ebb639faf5686f63b_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209593dea31f5494c3ebb639faf5686f63b_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,289 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097c553ef5b3b94558b48794860996b2ac_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097c553ef5b3b94558b48794860996b2ac_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,290 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098ad4aeee29234666a8da9a5dac013c33_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098ad4aeee29234666a8da9a5dac013c33_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,291 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab6b4e60fe03453fb2d167facc196582_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab6b4e60fe03453fb2d167facc196582_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,292 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209af3ed6372119402284a27d3bcf98395b_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209af3ed6372119402284a27d3bcf98395b_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,293 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bc94c8c7642e4c8e9371863977da0d4b_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bc94c8c7642e4c8e9371863977da0d4b_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,294 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c9dcaf05164746afb809bdc36c2a1e9f_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c9dcaf05164746afb809bdc36c2a1e9f_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,295 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d61bb5a5ce814f07b028f9e8b1f822ee_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d61bb5a5ce814f07b028f9e8b1f822ee_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,296 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f098817bc2fe4169bdf743095a3e11c3_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f098817bc2fe4169bdf743095a3e11c3_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,297 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f1d535b9d18b4d80a84b90271c388f28_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f1d535b9d18b4d80a84b90271c388f28_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,298 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209fcbd89efd429452fa2b0e4d16e2f779e_c7804971d6cc1c0c364cf6c88d0c46fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209fcbd89efd429452fa2b0e4d16e2f779e_c7804971d6cc1c0c364cf6c88d0c46fe 2024-12-09T20:42:46,299 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T20:42:46,300 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:46,303 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T20:42:46,305 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T20:42:46,306 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:46,306 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T20:42:46,306 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733776966306"}]},"ts":"9223372036854775807"} 2024-12-09T20:42:46,308 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T20:42:46,308 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c7804971d6cc1c0c364cf6c88d0c46fe, NAME => 'TestAcidGuarantees,,1733776932326.c7804971d6cc1c0c364cf6c88d0c46fe.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T20:42:46,308 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T20:42:46,308 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733776966308"}]},"ts":"9223372036854775807"} 2024-12-09T20:42:46,310 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T20:42:46,311 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:46,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-12-09T20:42:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T20:42:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T20:42:46,368 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:46,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 247) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_421848342_22 at /127.0.0.1:57406 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=455 (was 340) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3743 (was 3066) - AvailableMemoryMB LEAK? - 2024-12-09T20:42:46,384 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=455, ProcessCount=11, AvailableMemoryMB=3739 2024-12-09T20:42:46,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T20:42:46,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:42:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T20:42:46,387 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:42:46,387 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:46,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-12-09T20:42:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-09T20:42:46,388 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:42:46,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742055_1231 (size=1061) 2024-12-09T20:42:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-09T20:42:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-09T20:42:46,796 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948 2024-12-09T20:42:46,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742056_1232 (size=53) 2024-12-09T20:42:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-09T20:42:47,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:42:47,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d1100dd05f3cf346719eddc5a46c4983, disabling compactions & flushes 2024-12-09T20:42:47,205 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:47,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:47,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. after waiting 0 ms 2024-12-09T20:42:47,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:47,205 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:47,206 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1100dd05f3cf346719eddc5a46c4983: Waiting for close lock at 1733776967205Disabling compacts and flushes for region at 1733776967205Disabling writes for close at 1733776967205Writing region close event to WAL at 1733776967205Closed at 1733776967205 2024-12-09T20:42:47,207 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:42:47,207 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733776967207"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776967207"}]},"ts":"1733776967207"} 2024-12-09T20:42:47,209 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:42:47,210 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:42:47,210 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776967210"}]},"ts":"1733776967210"} 2024-12-09T20:42:47,212 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T20:42:47,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1100dd05f3cf346719eddc5a46c4983, ASSIGN}] 2024-12-09T20:42:47,214 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1100dd05f3cf346719eddc5a46c4983, ASSIGN 2024-12-09T20:42:47,215 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1100dd05f3cf346719eddc5a46c4983, ASSIGN; state=OFFLINE, location=00c23a7251f8,36519,1733776895755; forceNewPlan=false, retain=false 2024-12-09T20:42:47,366 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=d1100dd05f3cf346719eddc5a46c4983, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:47,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1100dd05f3cf346719eddc5a46c4983, ASSIGN because future has completed 2024-12-09T20:42:47,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:42:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-09T20:42:47,524 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:47,524 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:42:47,524 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,524 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:42:47,524 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,524 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,526 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,529 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:42:47,530 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1100dd05f3cf346719eddc5a46c4983 columnFamilyName A 2024-12-09T20:42:47,530 DEBUG [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:47,530 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] regionserver.HStore(327): Store=d1100dd05f3cf346719eddc5a46c4983/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:42:47,530 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,531 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:42:47,531 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1100dd05f3cf346719eddc5a46c4983 columnFamilyName B 2024-12-09T20:42:47,532 DEBUG [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:47,532 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] regionserver.HStore(327): Store=d1100dd05f3cf346719eddc5a46c4983/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:42:47,532 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,533 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:42:47,533 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1100dd05f3cf346719eddc5a46c4983 columnFamilyName C 2024-12-09T20:42:47,533 DEBUG [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:42:47,534 INFO [StoreOpener-d1100dd05f3cf346719eddc5a46c4983-1 {}] regionserver.HStore(327): Store=d1100dd05f3cf346719eddc5a46c4983/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:42:47,534 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:47,534 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,534 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,535 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,536 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,536 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,536 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:42:47,538 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,540 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:42:47,541 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened d1100dd05f3cf346719eddc5a46c4983; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59400482, jitterRate=-0.11486384272575378}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:42:47,541 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:47,543 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for d1100dd05f3cf346719eddc5a46c4983: Running coprocessor pre-open hook at 1733776967525Writing region info on filesystem at 1733776967525Initializing all the Stores at 1733776967525Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776967526 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776967526Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776967526Cleaning up temporary data from old regions at 1733776967536 (+10 ms)Running coprocessor post-open hooks at 1733776967541 (+5 ms)Region opened successfully at 1733776967543 (+2 ms) 2024-12-09T20:42:47,544 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., pid=63, masterSystemTime=1733776967520 2024-12-09T20:42:47,547 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:47,547 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:47,553 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=d1100dd05f3cf346719eddc5a46c4983, regionState=OPEN, openSeqNum=2, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:47,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:42:47,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-09T20:42:47,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 in 188 msec 2024-12-09T20:42:47,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-09T20:42:47,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1100dd05f3cf346719eddc5a46c4983, ASSIGN in 344 msec 2024-12-09T20:42:47,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:42:47,559 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776967559"}]},"ts":"1733776967559"} 2024-12-09T20:42:47,560 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T20:42:47,561 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:42:47,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-12-09T20:42:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-09T20:42:48,532 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:48,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,535 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f193499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,537 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bce0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,538 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,541 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T20:42:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c23a7251f8,42351,1733776895022 2024-12-09T20:42:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@601d0659 2024-12-09T20:42:48,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T20:42:48,545 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T20:42:48,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394f0213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b59cdd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,551 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50529ac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,552 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536e11ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,554 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e5ede0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,555 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7da37e6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,557 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d24abf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ec52c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,561 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a209190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,562 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@731f603a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,565 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563ce587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224ff965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,568 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230736d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,570 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1065a745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,572 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6585d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,574 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129ea183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,576 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b3100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,577 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f40068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:42:48,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:42:48,579 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:42:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:42:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:42:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37d0cc94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:42:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:42:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:42:48,581 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:42:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@889075a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:42:48,585 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,585 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,585 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,586 DEBUG [Thread-1100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,586 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,586 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,586 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-09T20:42:48,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T20:42:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,592 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:48,594 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:48,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:48,594 DEBUG [Thread-1091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,595 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,596 DEBUG [Thread-1091 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,597 DEBUG [Thread-1091 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,599 DEBUG [Thread-1089 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,599 DEBUG [Thread-1089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,605 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,614 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:42:48,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:42:48,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:42:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:48,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T20:42:48,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:48,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:48,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:48,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:42:48,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:48,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:48,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:42:48,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777028640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777028645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777028646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777028649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777028647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/177c56a4a2214230a8c1360a1ac329e9 is 50, key is test_row_0/A:col10/1733776968616/Put/seqid=0 2024-12-09T20:42:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742057_1233 (size=12001) 2024-12-09T20:42:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T20:42:48,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/177c56a4a2214230a8c1360a1ac329e9 2024-12-09T20:42:48,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:48,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:48,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:48,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/cb0774cb37a14464a459cee08d0b9308 is 50, key is test_row_0/B:col10/1733776968616/Put/seqid=0 2024-12-09T20:42:48,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777028761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777028761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777028761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777028762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777028771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742058_1234 (size=12001) 2024-12-09T20:42:48,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/cb0774cb37a14464a459cee08d0b9308 2024-12-09T20:42:48,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/5a57dcfd4b5a44adb7b0e03e535cc630 is 50, key is test_row_0/C:col10/1733776968616/Put/seqid=0 2024-12-09T20:42:48,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:48,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T20:42:48,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742059_1235 (size=12001) 2024-12-09T20:42:48,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/5a57dcfd4b5a44adb7b0e03e535cc630 2024-12-09T20:42:48,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/177c56a4a2214230a8c1360a1ac329e9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/177c56a4a2214230a8c1360a1ac329e9 2024-12-09T20:42:48,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/177c56a4a2214230a8c1360a1ac329e9, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T20:42:48,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/cb0774cb37a14464a459cee08d0b9308 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/cb0774cb37a14464a459cee08d0b9308 2024-12-09T20:42:48,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/cb0774cb37a14464a459cee08d0b9308, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T20:42:48,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/5a57dcfd4b5a44adb7b0e03e535cc630 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5a57dcfd4b5a44adb7b0e03e535cc630 2024-12-09T20:42:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777028971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777028971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777028971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777028972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:48,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5a57dcfd4b5a44adb7b0e03e535cc630, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T20:42:48,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d1100dd05f3cf346719eddc5a46c4983 in 366ms, sequenceid=13, compaction requested=false 2024-12-09T20:42:48,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:48,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776968644, max timestamp=1733776968984Num uniques 859; 2024-12-09T20:42:48,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776968644, max timestamp=1733776968984Num uniques 859; 2024-12-09T20:42:48,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:48,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:48,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776968644, max timestamp=1733776968984Num uniques 859; 2024-12-09T20:42:48,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:48,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-12-09T20:42:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:48,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/7018632325ee4c8986d350c7161f9b6e is 50, key is test_row_0/A:col10/1733776968647/Put/seqid=0 2024-12-09T20:42:49,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742060_1236 (size=12001) 2024-12-09T20:42:49,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776969073, max timestamp=1733776969102Num uniques 859; 2024-12-09T20:42:49,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:49,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776969073, max timestamp=1733776969102Num uniques 859; 2024-12-09T20:42:49,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:49,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776969073, max timestamp=1733776969102Num uniques 859; 2024-12-09T20:42:49,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:42:49,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:49,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:49,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T20:42:49,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733776969204, max timestamp=1733776969221Num uniques 1018; 2024-12-09T20:42:49,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:42:49,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733776969204, max timestamp=1733776969221Num uniques 1018; 2024-12-09T20:42:49,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733776969204, max timestamp=1733776969221Num uniques 1018; 2024-12-09T20:42:49,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:42:49,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:42:49,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733776969281, max timestamp=1733776969289Num uniques 1027; 2024-12-09T20:42:49,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T20:42:49,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733776969281, max timestamp=1733776969289Num uniques 1027; 2024-12-09T20:42:49,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T20:42:49,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733776969281, max timestamp=1733776969289Num uniques 1027; 2024-12-09T20:42:49,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T20:42:49,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13089, totalHeapSize=29984, min timestamp=1733776969299, max timestamp=1733776969316Num uniques 1036; 2024-12-09T20:42:49,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733776969299, max timestamp=1733776969316Num uniques 1036; 2024-12-09T20:42:49,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13086, totalHeapSize=29984, min timestamp=1733776969299, max timestamp=1733776969316Num uniques 1036; 2024-12-09T20:42:49,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47439, new segment data size=13089, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-09T20:42:49,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=13086, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-09T20:42:49,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-09T20:42:49,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733776969332, max timestamp=1733776969343Num uniques 1145; 2024-12-09T20:42:49,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-09T20:42:49,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733776969332, max timestamp=1733776969343Num uniques 1145; 2024-12-09T20:42:49,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-09T20:42:49,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733776969332, max timestamp=1733776969343Num uniques 1145; 2024-12-09T20:42:49,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-09T20:42:49,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:49,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:49,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733776969377, max timestamp=1733776969386Num uniques 1054; 2024-12-09T20:42:49,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T20:42:49,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733776969377, max timestamp=1733776969386Num uniques 1054; 2024-12-09T20:42:49,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T20:42:49,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733776969377, max timestamp=1733776969386Num uniques 1054; 2024-12-09T20:42:49,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T20:42:49,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/7018632325ee4c8986d350c7161f9b6e 2024-12-09T20:42:49,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733776969409, max timestamp=1733776969437Num uniques 1063; 2024-12-09T20:42:49,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T20:42:49,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733776969409, max timestamp=1733776969437Num uniques 1063; 2024-12-09T20:42:49,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T20:42:49,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733776969409, max timestamp=1733776969437Num uniques 1063; 2024-12-09T20:42:49,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T20:42:49,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/146b17c3e6744d60bc98f41394a36acd is 50, key is test_row_0/B:col10/1733776968647/Put/seqid=0 2024-12-09T20:42:49,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733776969477, max timestamp=1733776969490Num uniques 1072; 2024-12-09T20:42:49,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-12-09T20:42:49,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733776969477, max timestamp=1733776969490Num uniques 1072; 2024-12-09T20:42:49,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733776969477, max timestamp=1733776969490Num uniques 1072; 2024-12-09T20:42:49,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T20:42:49,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T20:42:49,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742061_1237 (size=12001) 2024-12-09T20:42:49,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733776969509, max timestamp=1733776969518Num uniques 1131; 2024-12-09T20:42:49,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-09T20:42:49,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733776969509, max timestamp=1733776969518Num uniques 1031; 2024-12-09T20:42:49,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733776969509, max timestamp=1733776969518Num uniques 1031; 2024-12-09T20:42:49,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T20:42:49,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T20:42:49,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733776969539, max timestamp=1733776969549Num uniques 1090; 2024-12-09T20:42:49,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733776969539, max timestamp=1733776969549Num uniques 1090; 2024-12-09T20:42:49,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-09T20:42:49,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-09T20:42:49,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733776969539, max timestamp=1733776969549Num uniques 1090; 2024-12-09T20:42:49,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-09T20:42:49,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733776969573, max timestamp=1733776969586Num uniques 1099; 2024-12-09T20:42:49,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-09T20:42:49,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733776969573, max timestamp=1733776969585Num uniques 1099; 2024-12-09T20:42:49,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-09T20:42:49,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733776969573, max timestamp=1733776969585Num uniques 1099; 2024-12-09T20:42:49,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-09T20:42:49,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733776969605, max timestamp=1733776969613Num uniques 1058; 2024-12-09T20:42:49,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733776969605, max timestamp=1733776969613Num uniques 1058; 2024-12-09T20:42:49,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733776969605, max timestamp=1733776969613Num uniques 1058; 2024-12-09T20:42:49,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-12-09T20:42:49,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-12-09T20:42:49,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-12-09T20:42:49,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733776969626, max timestamp=1733776969640Num uniques 1117; 2024-12-09T20:42:49,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733776969626, max timestamp=1733776969640Num uniques 1117; 2024-12-09T20:42:49,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=7642, suffix heap size=144136,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=167 2024-12-09T20:42:49,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=7642, suffix heap size=144136,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=167 2024-12-09T20:42:49,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733776969626, max timestamp=1733776969640Num uniques 1117; 2024-12-09T20:42:49,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=7642, suffix heap size=144136,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=167 2024-12-09T20:42:49,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:49,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:49,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733776969662, max timestamp=1733776969674Num uniques 1026; 2024-12-09T20:42:49,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-09T20:42:49,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733776969662, max timestamp=1733776969674Num uniques 1026; 2024-12-09T20:42:49,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-09T20:42:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733776969662, max timestamp=1733776969674Num uniques 1026; 2024-12-09T20:42:49,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-09T20:42:49,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10750, totalHeapSize=24680, min timestamp=1733776969719, max timestamp=1733776969727Num uniques 1085; 2024-12-09T20:42:49,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49680, new segment data size=10750, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-12-09T20:42:49,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10750, totalHeapSize=24680, min timestamp=1733776969719, max timestamp=1733776969727Num uniques 1085; 2024-12-09T20:42:49,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10750, totalHeapSize=24680, min timestamp=1733776969719, max timestamp=1733776969727Num uniques 1085; 2024-12-09T20:42:49,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49680, new segment data size=10750, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-12-09T20:42:49,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49680, new segment data size=10750, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-12-09T20:42:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T20:42:49,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733776969743, max timestamp=1733776969745Num uniques 1094; 2024-12-09T20:42:49,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-12-09T20:42:49,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733776969743, max timestamp=1733776969745Num uniques 1094; 2024-12-09T20:42:49,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-12-09T20:42:49,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733776969743, max timestamp=1733776969745Num uniques 1094; 2024-12-09T20:42:49,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-12-09T20:42:49,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733776969752, max timestamp=1733776969766Num uniques 1103; 2024-12-09T20:42:49,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=7005, suffix heap size=142680,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=153 2024-12-09T20:42:49,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733776969752, max timestamp=1733776969766Num uniques 1103; 2024-12-09T20:42:49,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=7005, suffix heap size=142680,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=153 2024-12-09T20:42:49,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733776969752, max timestamp=1733776969766Num uniques 1103; 2024-12-09T20:42:49,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=7005, suffix heap size=142680,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=153 2024-12-09T20:42:49,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7410, totalHeapSize=17088, min timestamp=1733776969780, max timestamp=1733776969812Num uniques 1012; 2024-12-09T20:42:49,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7410, totalHeapSize=17088, min timestamp=1733776969780, max timestamp=1733776969812Num uniques 1012; 2024-12-09T20:42:49,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46340, new segment data size=7410, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-12-09T20:42:49,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46340, new segment data size=7410, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-12-09T20:42:49,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7410, totalHeapSize=17088, min timestamp=1733776969780, max timestamp=1733776969812Num uniques 1012; 2024-12-09T20:42:49,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46340, new segment data size=7410, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-12-09T20:42:49,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:49,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:49,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1733776969819, max timestamp=1733776969866Num uniques 1021; 2024-12-09T20:42:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10106, totalHeapSize=23224, min timestamp=1733776969819, max timestamp=1733776969867Num uniques 1021; 2024-12-09T20:42:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1733776969819, max timestamp=1733776969866Num uniques 1021; 2024-12-09T20:42:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-12-09T20:42:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46746, new segment data size=10106, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-12-09T20:42:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-12-09T20:42:49,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1733776969881, max timestamp=1733776969895Num uniques 1080; 2024-12-09T20:42:49,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=12820, suffix heap size=140288,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=280 2024-12-09T20:42:49,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1733776969881, max timestamp=1733776969895Num uniques 1080; 2024-12-09T20:42:49,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=12820, suffix heap size=140288,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=280 2024-12-09T20:42:49,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733776969886, max timestamp=1733776969895Num uniques 1080; 2024-12-09T20:42:49,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=6870, suffix heap size=140288,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=150 2024-12-09T20:42:49,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/146b17c3e6744d60bc98f41394a36acd 2024-12-09T20:42:49,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/75dd46cd05c74fb78d15c8ac35d63302 is 50, key is test_row_0/C:col10/1733776968647/Put/seqid=0 2024-12-09T20:42:49,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1733776969903, max timestamp=1733776969920Num uniques 1139; 2024-12-09T20:42:49,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1733776969903, max timestamp=1733776969920Num uniques 1139; 2024-12-09T20:42:49,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=8654, suffix heap size=146424,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=189 2024-12-09T20:42:49,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=8654, suffix heap size=146424,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=189 2024-12-09T20:42:49,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1733776969903, max timestamp=1733776969920Num uniques 1009; 2024-12-09T20:42:49,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8654, suffix heap size=132904,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=189 2024-12-09T20:42:49,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742062_1238 (size=12001) 2024-12-09T20:42:49,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1733776969932, max timestamp=1733776969941Num uniques 1048; 2024-12-09T20:42:49,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47998, new segment data size=11358, suffix heap size=136960,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=248 2024-12-09T20:42:49,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1733776969932, max timestamp=1733776969941Num uniques 1048; 2024-12-09T20:42:49,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1733776969932, max timestamp=1733776969941Num uniques 1048; 2024-12-09T20:42:49,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47998, new segment data size=11358, suffix heap size=136960,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=248 2024-12-09T20:42:49,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47998, new segment data size=11358, suffix heap size=136960,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=248 2024-12-09T20:42:49,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=257, cellSize=11765, totalHeapSize=26968, min timestamp=1733776969955, max timestamp=1733776969966Num uniques 1107; 2024-12-09T20:42:49,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=11765, suffix heap size=143096,new segment heap size=26968 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=257 2024-12-09T20:42:49,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=257, cellSize=11765, totalHeapSize=26968, min timestamp=1733776969955, max timestamp=1733776969966Num uniques 1107; 2024-12-09T20:42:49,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=11765, suffix heap size=143096,new segment heap size=26968 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=257 2024-12-09T20:42:49,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=257, cellSize=11765, totalHeapSize=26968, min timestamp=1733776969955, max timestamp=1733776969966Num uniques 1107; 2024-12-09T20:42:49,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=11765, suffix heap size=143096,new segment heap size=26968 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=257 2024-12-09T20:42:49,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777029988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:49,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733776969979, max timestamp=1733776969990Num uniques 1116; 2024-12-09T20:42:49,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51106, new segment data size=9886, suffix heap size=144032,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1116, new segment cells count=216 2024-12-09T20:42:49,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733776969979, max timestamp=1733776969990Num uniques 1116; 2024-12-09T20:42:49,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51106, new segment data size=9886, suffix heap size=144032,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1116, new segment cells count=216 2024-12-09T20:42:49,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733776969979, max timestamp=1733776969990Num uniques 1116; 2024-12-09T20:42:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:49,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51106, new segment data size=9886, suffix heap size=144032,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1116, new segment cells count=216 2024-12-09T20:42:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:49,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:49,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:49,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:49,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777030029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=7996, totalHeapSize=18440, min timestamp=1733776970008, max timestamp=1733776970033Num uniques 1075; 2024-12-09T20:42:50,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1733776970008, max timestamp=1733776970033Num uniques 1075; 2024-12-09T20:42:50,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49216, new segment data size=7996, suffix heap size=139768,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=175 2024-12-09T20:42:50,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=8010, suffix heap size=139768,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=175 2024-12-09T20:42:50,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8001, totalHeapSize=18440, min timestamp=1733776970008, max timestamp=1733776970033Num uniques 1075; 2024-12-09T20:42:50,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49221, new segment data size=8001, suffix heap size=139768,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=175 2024-12-09T20:42:50,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1733776970078, max timestamp=1733776970099Num uniques 1034; 2024-12-09T20:42:50,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10704, totalHeapSize=24576, min timestamp=1733776970078, max timestamp=1733776970100Num uniques 1034; 2024-12-09T20:42:50,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1733776970078, max timestamp=1733776970099Num uniques 1034; 2024-12-09T20:42:50,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47354, new segment data size=10714, suffix heap size=135504,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1034, new segment cells count=234 2024-12-09T20:42:50,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47344, new segment data size=10704, suffix heap size=135504,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1034, new segment cells count=234 2024-12-09T20:42:50,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47354, new segment data size=10714, suffix heap size=135504,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1034, new segment cells count=234 2024-12-09T20:42:50,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=193, cellSize=8828, totalHeapSize=20312, min timestamp=1733776970120, max timestamp=1733776970126Num uniques 1093; 2024-12-09T20:42:50,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50048, new segment data size=8828, suffix heap size=141640,new segment heap size=20312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=193 2024-12-09T20:42:50,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=193, cellSize=8828, totalHeapSize=20312, min timestamp=1733776970120, max timestamp=1733776970126Num uniques 1093; 2024-12-09T20:42:50,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50048, new segment data size=8828, suffix heap size=141640,new segment heap size=20312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=193 2024-12-09T20:42:50,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=193, cellSize=8828, totalHeapSize=20312, min timestamp=1733776970120, max timestamp=1733776970126Num uniques 1093; 2024-12-09T20:42:50,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50048, new segment data size=8828, suffix heap size=141640,new segment heap size=20312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=193 2024-12-09T20:42:50,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:50,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:50,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=202, cellSize=9232, totalHeapSize=21248, min timestamp=1733776970144, max timestamp=1733776970159Num uniques 1052; 2024-12-09T20:42:50,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48162, new segment data size=9232, suffix heap size=137376,new segment heap size=21248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1052, new segment cells count=202 2024-12-09T20:42:50,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=202, cellSize=9232, totalHeapSize=21248, min timestamp=1733776970144, max timestamp=1733776970159Num uniques 1052; 2024-12-09T20:42:50,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48162, new segment data size=9232, suffix heap size=137376,new segment heap size=21248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1052, new segment cells count=202 2024-12-09T20:42:50,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=202, cellSize=9235, totalHeapSize=21248, min timestamp=1733776970144, max timestamp=1733776970159Num uniques 1052; 2024-12-09T20:42:50,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48165, new segment data size=9235, suffix heap size=137376,new segment heap size=21248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1052, new segment cells count=202 2024-12-09T20:42:50,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777030181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7365, totalHeapSize=16984, min timestamp=1733776970174, max timestamp=1733776970185Num uniques 1061; 2024-12-09T20:42:50,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7366, totalHeapSize=16984, min timestamp=1733776970174, max timestamp=1733776970185Num uniques 1061; 2024-12-09T20:42:50,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48585, new segment data size=7365, suffix heap size=138312,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1061, new segment cells count=161 2024-12-09T20:42:50,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48586, new segment data size=7366, suffix heap size=138312,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1061, new segment cells count=161 2024-12-09T20:42:50,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7366, totalHeapSize=16984, min timestamp=1733776970174, max timestamp=1733776970185Num uniques 1061; 2024-12-09T20:42:50,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48586, new segment data size=7366, suffix heap size=138312,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1061, new segment cells count=161 2024-12-09T20:42:50,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=220, cellSize=10070, totalHeapSize=23120, min timestamp=1733776970205, max timestamp=1733776970210Num uniques 1020; 2024-12-09T20:42:50,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46710, new segment data size=10070, suffix heap size=134048,new segment heap size=23120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=220 2024-12-09T20:42:50,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=220, cellSize=10070, totalHeapSize=23120, min timestamp=1733776970205, max timestamp=1733776970210Num uniques 1020; 2024-12-09T20:42:50,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46710, new segment data size=10070, suffix heap size=134048,new segment heap size=23120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=220 2024-12-09T20:42:50,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=220, cellSize=10070, totalHeapSize=23120, min timestamp=1733776970205, max timestamp=1733776970210Num uniques 1020; 2024-12-09T20:42:50,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46710, new segment data size=10070, suffix heap size=134048,new segment heap size=23120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=220 2024-12-09T20:42:50,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=179, cellSize=8194, totalHeapSize=18856, min timestamp=1733776970231, max timestamp=1733776970236Num uniques 1079; 2024-12-09T20:42:50,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=179, cellSize=8194, totalHeapSize=18856, min timestamp=1733776970231, max timestamp=1733776970236Num uniques 1079; 2024-12-09T20:42:50,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49414, new segment data size=8194, suffix heap size=140184,new segment heap size=18856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1079, new segment cells count=179 2024-12-09T20:42:50,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=179, cellSize=8194, totalHeapSize=18856, min timestamp=1733776970231, max timestamp=1733776970236Num uniques 1079; 2024-12-09T20:42:50,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49414, new segment data size=8194, suffix heap size=140184,new segment heap size=18856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1079, new segment cells count=179 2024-12-09T20:42:50,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49414, new segment data size=8194, suffix heap size=140184,new segment heap size=18856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1079, new segment cells count=179 2024-12-09T20:42:50,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=188, cellSize=8608, totalHeapSize=19792, min timestamp=1733776970246, max timestamp=1733776970256Num uniques 1038; 2024-12-09T20:42:50,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=188, cellSize=8608, totalHeapSize=19792, min timestamp=1733776970246, max timestamp=1733776970256Num uniques 1038; 2024-12-09T20:42:50,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47538, new segment data size=8608, suffix heap size=135920,new segment heap size=19792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1038, new segment cells count=188 2024-12-09T20:42:50,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47538, new segment data size=8608, suffix heap size=135920,new segment heap size=19792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1038, new segment cells count=188 2024-12-09T20:42:50,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=188, cellSize=8608, totalHeapSize=19792, min timestamp=1733776970246, max timestamp=1733776970256Num uniques 1038; 2024-12-09T20:42:50,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47538, new segment data size=8608, suffix heap size=135920,new segment heap size=19792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1038, new segment cells count=188 2024-12-09T20:42:50,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1733776970268, max timestamp=1733776970279Num uniques 1047; 2024-12-09T20:42:50,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47952, new segment data size=9022, suffix heap size=136856,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1047, new segment cells count=197 2024-12-09T20:42:50,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1733776970268, max timestamp=1733776970279Num uniques 1047; 2024-12-09T20:42:50,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1733776970268, max timestamp=1733776970279Num uniques 1047; 2024-12-09T20:42:50,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47952, new segment data size=9022, suffix heap size=136856,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1047, new segment cells count=197 2024-12-09T20:42:50,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47952, new segment data size=9022, suffix heap size=136856,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1047, new segment cells count=197 2024-12-09T20:42:50,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=156, cellSize=7140, totalHeapSize=16464, min timestamp=1733776970300, max timestamp=1733776970308Num uniques 1056; 2024-12-09T20:42:50,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=7140, suffix heap size=137792,new segment heap size=16464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=156 2024-12-09T20:42:50,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=156, cellSize=7140, totalHeapSize=16464, min timestamp=1733776970300, max timestamp=1733776970308Num uniques 1056; 2024-12-09T20:42:50,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=7140, suffix heap size=137792,new segment heap size=16464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=156 2024-12-09T20:42:50,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=156, cellSize=7140, totalHeapSize=16464, min timestamp=1733776970300, max timestamp=1733776970308Num uniques 1056; 2024-12-09T20:42:50,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=7140, suffix heap size=137792,new segment heap size=16464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=156 2024-12-09T20:42:50,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=215, cellSize=9840, totalHeapSize=22600, min timestamp=1733776970323, max timestamp=1733776970330Num uniques 1015; 2024-12-09T20:42:50,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46480, new segment data size=9840, suffix heap size=133528,new segment heap size=22600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1015, new segment cells count=215 2024-12-09T20:42:50,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=215, cellSize=9840, totalHeapSize=22600, min timestamp=1733776970323, max timestamp=1733776970330Num uniques 1015; 2024-12-09T20:42:50,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=215, cellSize=9840, totalHeapSize=22600, min timestamp=1733776970323, max timestamp=1733776970330Num uniques 1015; 2024-12-09T20:42:50,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46480, new segment data size=9840, suffix heap size=133528,new segment heap size=22600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1015, new segment cells count=215 2024-12-09T20:42:50,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46480, new segment data size=9840, suffix heap size=133528,new segment heap size=22600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1015, new segment cells count=215 2024-12-09T20:42:50,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/75dd46cd05c74fb78d15c8ac35d63302 2024-12-09T20:42:50,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/7018632325ee4c8986d350c7161f9b6e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/7018632325ee4c8986d350c7161f9b6e 2024-12-09T20:42:50,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/7018632325ee4c8986d350c7161f9b6e, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T20:42:50,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/146b17c3e6744d60bc98f41394a36acd as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/146b17c3e6744d60bc98f41394a36acd 2024-12-09T20:42:50,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/146b17c3e6744d60bc98f41394a36acd, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T20:42:50,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/75dd46cd05c74fb78d15c8ac35d63302 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/75dd46cd05c74fb78d15c8ac35d63302 2024-12-09T20:42:50,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/75dd46cd05c74fb78d15c8ac35d63302, entries=150, sequenceid=34, filesize=11.7 K 2024-12-09T20:42:50,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=87.22 KB/89310 for d1100dd05f3cf346719eddc5a46c4983 in 1368ms, sequenceid=34, compaction requested=false 2024-12-09T20:42:50,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:50,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:50,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=107.34 KB heapSize=300.05 KB 2024-12-09T20:42:50,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:50,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:50,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:50,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:50,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:50,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:50,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/4cb1c3095d1a434dbcf190e61ec65e29 is 50, key is test_row_0/A:col10/1733776970346/Put/seqid=0 2024-12-09T20:42:50,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742063_1239 (size=12301) 2024-12-09T20:42:50,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/4cb1c3095d1a434dbcf190e61ec65e29 2024-12-09T20:42:50,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777030397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777030398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777030401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777030403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777030405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/6b78f0fc4d9d46448fcf5506fd900c6c is 50, key is test_row_0/B:col10/1733776970346/Put/seqid=0 2024-12-09T20:42:50,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742064_1240 (size=12301) 2024-12-09T20:42:50,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:50,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:50,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777030511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777030511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777030511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777030511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777030521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777030721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777030721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777030722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777030721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777030731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T20:42:50,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/6b78f0fc4d9d46448fcf5506fd900c6c 2024-12-09T20:42:50,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/c3a5fdd3b5f84a8a9789a6989a86798f is 50, key is test_row_0/C:col10/1733776970346/Put/seqid=0 2024-12-09T20:42:50,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742065_1241 (size=12301) 2024-12-09T20:42:50,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:50,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:51,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777031032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777031035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777031035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777031041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777031045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:51,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:51,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:51,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:51,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:51,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:51,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/c3a5fdd3b5f84a8a9789a6989a86798f 2024-12-09T20:42:51,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/4cb1c3095d1a434dbcf190e61ec65e29 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/4cb1c3095d1a434dbcf190e61ec65e29 2024-12-09T20:42:51,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/4cb1c3095d1a434dbcf190e61ec65e29, entries=150, sequenceid=650, filesize=12.0 K 2024-12-09T20:42:51,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/6b78f0fc4d9d46448fcf5506fd900c6c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6b78f0fc4d9d46448fcf5506fd900c6c 2024-12-09T20:42:51,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6b78f0fc4d9d46448fcf5506fd900c6c, entries=150, sequenceid=650, filesize=12.0 K 2024-12-09T20:42:51,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/c3a5fdd3b5f84a8a9789a6989a86798f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c3a5fdd3b5f84a8a9789a6989a86798f 2024-12-09T20:42:51,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c3a5fdd3b5f84a8a9789a6989a86798f, entries=150, sequenceid=650, filesize=12.0 K 2024-12-09T20:42:51,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~300 KB/307200, currentSize=73.80 KB/75570 for d1100dd05f3cf346719eddc5a46c4983 in 972ms, sequenceid=650, compaction requested=true 2024-12-09T20:42:51,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:51,332 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:42:51,332 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:51,333 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36303 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:51,333 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/B is initiating minor compaction (all files) 2024-12-09T20:42:51,333 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/B in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,333 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/cb0774cb37a14464a459cee08d0b9308, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/146b17c3e6744d60bc98f41394a36acd, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6b78f0fc4d9d46448fcf5506fd900c6c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=35.5 K 2024-12-09T20:42:51,333 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36303 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:51,333 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/A is initiating minor compaction (all files) 2024-12-09T20:42:51,333 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/A in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,333 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/177c56a4a2214230a8c1360a1ac329e9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/7018632325ee4c8986d350c7161f9b6e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/4cb1c3095d1a434dbcf190e61ec65e29] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=35.5 K 2024-12-09T20:42:51,334 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 177c56a4a2214230a8c1360a1ac329e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733776968603 2024-12-09T20:42:51,334 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0774cb37a14464a459cee08d0b9308, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733776968603 2024-12-09T20:42:51,334 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7018632325ee4c8986d350c7161f9b6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733776968644 2024-12-09T20:42:51,335 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 146b17c3e6744d60bc98f41394a36acd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733776968644 2024-12-09T20:42:51,335 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b78f0fc4d9d46448fcf5506fd900c6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733776970346 2024-12-09T20:42:51,336 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cb1c3095d1a434dbcf190e61ec65e29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733776970346 2024-12-09T20:42:51,366 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#A#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:51,367 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/507ae0c5499e49d0bde93d65cb5506c3 is 50, key is test_row_0/A:col10/1733776970346/Put/seqid=0 2024-12-09T20:42:51,369 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#B#compaction#199 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:51,369 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/1bf061a90a5d4880acabc1680a04d3af is 50, key is test_row_0/B:col10/1733776970346/Put/seqid=0 2024-12-09T20:42:51,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T20:42:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,378 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T20:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:51,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/91c58cc18b174503b6b53b674f6e4cfc is 50, key is test_row_0/A:col10/1733776970403/Put/seqid=0 2024-12-09T20:42:51,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742066_1242 (size=12404) 2024-12-09T20:42:51,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742067_1243 (size=12404) 2024-12-09T20:42:51,427 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/1bf061a90a5d4880acabc1680a04d3af as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1bf061a90a5d4880acabc1680a04d3af 2024-12-09T20:42:51,435 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/B of d1100dd05f3cf346719eddc5a46c4983 into 1bf061a90a5d4880acabc1680a04d3af(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:51,435 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:51,435 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/B, priority=13, startTime=1733776971332; duration=0sec 2024-12-09T20:42:51,435 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:51,435 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:42:51,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742068_1244 (size=12301) 2024-12-09T20:42:51,435 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:51,438 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/91c58cc18b174503b6b53b674f6e4cfc 2024-12-09T20:42:51,439 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36303 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:51,439 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/C is initiating minor compaction (all files) 2024-12-09T20:42:51,439 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/C in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,440 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5a57dcfd4b5a44adb7b0e03e535cc630, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/75dd46cd05c74fb78d15c8ac35d63302, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c3a5fdd3b5f84a8a9789a6989a86798f] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=35.5 K 2024-12-09T20:42:51,441 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a57dcfd4b5a44adb7b0e03e535cc630, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733776968603 2024-12-09T20:42:51,442 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 75dd46cd05c74fb78d15c8ac35d63302, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733776968644 2024-12-09T20:42:51,444 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c3a5fdd3b5f84a8a9789a6989a86798f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733776970346 2024-12-09T20:42:51,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4d15fbf97d904078aeadc41aa27e838d is 50, key is test_row_0/B:col10/1733776970403/Put/seqid=0 2024-12-09T20:42:51,479 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#C#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:51,479 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/84b32028087249c89232f0335c888b7a is 50, key is test_row_0/C:col10/1733776970346/Put/seqid=0 2024-12-09T20:42:51,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742070_1246 (size=12404) 2024-12-09T20:42:51,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742069_1245 (size=12301) 2024-12-09T20:42:51,523 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4d15fbf97d904078aeadc41aa27e838d 2024-12-09T20:42:51,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/223f2a04c4564816abd5a4c2964a9938 is 50, key is test_row_0/C:col10/1733776970403/Put/seqid=0 2024-12-09T20:42:51,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:51,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742071_1247 (size=12301) 2024-12-09T20:42:51,561 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/223f2a04c4564816abd5a4c2964a9938 2024-12-09T20:42:51,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/91c58cc18b174503b6b53b674f6e4cfc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/91c58cc18b174503b6b53b674f6e4cfc 2024-12-09T20:42:51,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777031566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777031569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777031569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777031570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777031570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,577 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:42:51,578 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/91c58cc18b174503b6b53b674f6e4cfc, entries=150, sequenceid=664, filesize=12.0 K 2024-12-09T20:42:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4d15fbf97d904078aeadc41aa27e838d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4d15fbf97d904078aeadc41aa27e838d 2024-12-09T20:42:51,594 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4d15fbf97d904078aeadc41aa27e838d, entries=150, sequenceid=664, filesize=12.0 K 2024-12-09T20:42:51,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/223f2a04c4564816abd5a4c2964a9938 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/223f2a04c4564816abd5a4c2964a9938 2024-12-09T20:42:51,600 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/223f2a04c4564816abd5a4c2964a9938, entries=150, sequenceid=664, filesize=12.0 K 2024-12-09T20:42:51,601 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d1100dd05f3cf346719eddc5a46c4983 in 224ms, sequenceid=664, compaction requested=false 2024-12-09T20:42:51,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:51,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-09T20:42:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-09T20:42:51,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-09T20:42:51,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0090 sec 2024-12-09T20:42:51,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 3.0200 sec 2024-12-09T20:42:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:51,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:42:51,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:51,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:51,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:51,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:51,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:51,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:51,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777031695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777031698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777031698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777031699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777031700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/9849d92176d54352871539c84a3fe250 is 50, key is test_row_0/A:col10/1733776971682/Put/seqid=0 2024-12-09T20:42:51,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742072_1248 (size=12301) 2024-12-09T20:42:51,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/9849d92176d54352871539c84a3fe250 2024-12-09T20:42:51,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/57ecd69a29ad40788dfee8f21c11704a is 50, key is test_row_0/B:col10/1733776971682/Put/seqid=0 2024-12-09T20:42:51,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742073_1249 (size=12301) 2024-12-09T20:42:51,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/57ecd69a29ad40788dfee8f21c11704a 2024-12-09T20:42:51,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/e580ffc8736f413586817ca934a3aa10 is 50, key is test_row_0/C:col10/1733776971682/Put/seqid=0 2024-12-09T20:42:51,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742074_1250 (size=12301) 2024-12-09T20:42:51,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/e580ffc8736f413586817ca934a3aa10 2024-12-09T20:42:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777031812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777031812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777031812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777031812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777031812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:51,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/9849d92176d54352871539c84a3fe250 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9849d92176d54352871539c84a3fe250 2024-12-09T20:42:51,821 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/507ae0c5499e49d0bde93d65cb5506c3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/507ae0c5499e49d0bde93d65cb5506c3 2024-12-09T20:42:51,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9849d92176d54352871539c84a3fe250, entries=150, sequenceid=684, filesize=12.0 K 2024-12-09T20:42:51,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/57ecd69a29ad40788dfee8f21c11704a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/57ecd69a29ad40788dfee8f21c11704a 2024-12-09T20:42:51,828 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/A of d1100dd05f3cf346719eddc5a46c4983 into 507ae0c5499e49d0bde93d65cb5506c3(size=12.1 K), total size for store is 36.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:51,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:51,828 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/A, priority=13, startTime=1733776971331; duration=0sec 2024-12-09T20:42:51,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:51,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:42:51,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/57ecd69a29ad40788dfee8f21c11704a, entries=150, sequenceid=684, filesize=12.0 K 2024-12-09T20:42:51,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/e580ffc8736f413586817ca934a3aa10 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/e580ffc8736f413586817ca934a3aa10 2024-12-09T20:42:51,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/e580ffc8736f413586817ca934a3aa10, entries=150, sequenceid=684, filesize=12.0 K 2024-12-09T20:42:51,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d1100dd05f3cf346719eddc5a46c4983 in 157ms, sequenceid=684, compaction requested=true 2024-12-09T20:42:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:51,840 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:42:51,841 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37006 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:51,842 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/A is initiating minor compaction (all files) 2024-12-09T20:42:51,842 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/A in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,842 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/507ae0c5499e49d0bde93d65cb5506c3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/91c58cc18b174503b6b53b674f6e4cfc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9849d92176d54352871539c84a3fe250] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.1 K 2024-12-09T20:42:51,842 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 507ae0c5499e49d0bde93d65cb5506c3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733776970346 2024-12-09T20:42:51,843 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c58cc18b174503b6b53b674f6e4cfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1733776970391 2024-12-09T20:42:51,843 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9849d92176d54352871539c84a3fe250, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733776971566 2024-12-09T20:42:51,858 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#A#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:51,859 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/ff98396afbf5422eb2edcab6742131d6 is 50, key is test_row_0/A:col10/1733776971682/Put/seqid=0 2024-12-09T20:42:51,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742075_1251 (size=12507) 2024-12-09T20:42:51,901 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/ff98396afbf5422eb2edcab6742131d6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ff98396afbf5422eb2edcab6742131d6 2024-12-09T20:42:51,910 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/A of d1100dd05f3cf346719eddc5a46c4983 into ff98396afbf5422eb2edcab6742131d6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:51,910 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:51,910 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/A, priority=13, startTime=1733776971840; duration=0sec 2024-12-09T20:42:51,910 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:42:51,910 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:42:51,910 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-12-09T20:42:51,911 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T20:42:51,911 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T20:42:51,911 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. because compaction request was cancelled 2024-12-09T20:42:51,911 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:42:51,911 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:51,912 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37006 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:51,912 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/B is initiating minor compaction (all files) 2024-12-09T20:42:51,912 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/B in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:51,912 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1bf061a90a5d4880acabc1680a04d3af, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4d15fbf97d904078aeadc41aa27e838d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/57ecd69a29ad40788dfee8f21c11704a] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.1 K 2024-12-09T20:42:51,913 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bf061a90a5d4880acabc1680a04d3af, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733776970346 2024-12-09T20:42:51,914 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d15fbf97d904078aeadc41aa27e838d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1733776970391 2024-12-09T20:42:51,914 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ecd69a29ad40788dfee8f21c11704a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733776971566 2024-12-09T20:42:51,926 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:51,926 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/d7b2cb4e78dd4bebaa5f16624930f4bb is 50, key is test_row_0/B:col10/1733776971682/Put/seqid=0 2024-12-09T20:42:51,927 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/84b32028087249c89232f0335c888b7a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/84b32028087249c89232f0335c888b7a 2024-12-09T20:42:51,934 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/C of d1100dd05f3cf346719eddc5a46c4983 into 84b32028087249c89232f0335c888b7a(size=12.1 K), total size for store is 36.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:51,934 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:51,935 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/C, priority=13, startTime=1733776971332; duration=0sec 2024-12-09T20:42:51,935 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:51,935 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:42:51,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742076_1252 (size=12507) 2024-12-09T20:42:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:52,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T20:42:52,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:52,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:52,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:52,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:52,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:52,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:52,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/661c42a43e064233bdad88970a363b80 is 50, key is test_row_0/A:col10/1733776972024/Put/seqid=0 2024-12-09T20:42:52,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742077_1253 (size=12301) 2024-12-09T20:42:52,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=703 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/661c42a43e064233bdad88970a363b80 2024-12-09T20:42:52,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3deb7337808e441b8a1d6bbdc118133d is 50, key is test_row_0/B:col10/1733776972024/Put/seqid=0 2024-12-09T20:42:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777032047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742078_1254 (size=12301) 2024-12-09T20:42:52,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777032049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777032051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777032052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777032052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777032161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777032161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777032162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777032162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777032162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,342 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/d7b2cb4e78dd4bebaa5f16624930f4bb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/d7b2cb4e78dd4bebaa5f16624930f4bb 2024-12-09T20:42:52,346 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/B of d1100dd05f3cf346719eddc5a46c4983 into d7b2cb4e78dd4bebaa5f16624930f4bb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:52,347 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:52,347 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/B, priority=13, startTime=1733776971840; duration=0sec 2024-12-09T20:42:52,347 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:52,347 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:42:52,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777032371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777032371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777032371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777032371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777032371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=703 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3deb7337808e441b8a1d6bbdc118133d 2024-12-09T20:42:52,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/ae03f45742ff4ccf87cba73d11406e7a is 50, key is test_row_0/C:col10/1733776972024/Put/seqid=0 2024-12-09T20:42:52,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742079_1255 (size=12301) 2024-12-09T20:42:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777032682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777032682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777032682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777032683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777032683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T20:42:52,761 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:52,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-09T20:42:52,766 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T20:42:52,766 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:52,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T20:42:52,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T20:42:52,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:52,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:52,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:52,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:52,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:52,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=703 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/ae03f45742ff4ccf87cba73d11406e7a 2024-12-09T20:42:52,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/661c42a43e064233bdad88970a363b80 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/661c42a43e064233bdad88970a363b80 2024-12-09T20:42:52,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/661c42a43e064233bdad88970a363b80, entries=150, sequenceid=703, filesize=12.0 K 2024-12-09T20:42:52,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3deb7337808e441b8a1d6bbdc118133d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3deb7337808e441b8a1d6bbdc118133d 2024-12-09T20:42:52,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3deb7337808e441b8a1d6bbdc118133d, entries=150, sequenceid=703, filesize=12.0 K 2024-12-09T20:42:52,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/ae03f45742ff4ccf87cba73d11406e7a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/ae03f45742ff4ccf87cba73d11406e7a 2024-12-09T20:42:52,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/ae03f45742ff4ccf87cba73d11406e7a, entries=150, sequenceid=703, filesize=12.0 K 2024-12-09T20:42:52,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d1100dd05f3cf346719eddc5a46c4983 in 953ms, sequenceid=703, compaction requested=true 2024-12-09T20:42:52,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:52,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:52,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:52,978 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T20:42:52,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:52,978 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T20:42:52,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:52,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:52,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:52,979 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T20:42:52,979 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T20:42:52,979 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T20:42:52,979 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T20:42:52,979 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. because compaction request was cancelled 2024-12-09T20:42:52,979 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. because compaction request was cancelled 2024-12-09T20:42:52,979 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:42:52,979 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:42:52,979 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:52,981 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49307 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:52,981 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/C is initiating minor compaction (all files) 2024-12-09T20:42:52,981 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/C in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:52,982 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/84b32028087249c89232f0335c888b7a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/223f2a04c4564816abd5a4c2964a9938, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/e580ffc8736f413586817ca934a3aa10, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/ae03f45742ff4ccf87cba73d11406e7a] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=48.2 K 2024-12-09T20:42:52,983 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84b32028087249c89232f0335c888b7a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733776970346 2024-12-09T20:42:52,985 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 223f2a04c4564816abd5a4c2964a9938, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1733776970391 2024-12-09T20:42:52,985 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e580ffc8736f413586817ca934a3aa10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733776971566 2024-12-09T20:42:52,986 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae03f45742ff4ccf87cba73d11406e7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1733776971697 2024-12-09T20:42:53,007 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:53,008 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/58349ea7aed9475cbfc6f29445cda437 is 50, key is test_row_0/C:col10/1733776972024/Put/seqid=0 2024-12-09T20:42:53,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T20:42:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T20:42:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:53,081 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T20:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:53,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742080_1256 (size=12541) 2024-12-09T20:42:53,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/6243f728f7df4c3694bd3988d5e54a91 is 50, key is test_row_0/A:col10/1733776972049/Put/seqid=0 2024-12-09T20:42:53,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742081_1257 (size=12301) 2024-12-09T20:42:53,158 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/6243f728f7df4c3694bd3988d5e54a91 2024-12-09T20:42:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/a7c559ee41194bce9c5743683cc8dccd is 50, key is test_row_0/B:col10/1733776972049/Put/seqid=0 2024-12-09T20:42:53,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742082_1258 (size=12301) 2024-12-09T20:42:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:53,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:53,209 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/a7c559ee41194bce9c5743683cc8dccd 2024-12-09T20:42:53,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/2f744a0587644584a50deb11f0301b99 is 50, key is test_row_0/C:col10/1733776972049/Put/seqid=0 2024-12-09T20:42:53,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777033223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777033224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777033228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777033229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777033232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742083_1259 (size=12301) 2024-12-09T20:42:53,250 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/2f744a0587644584a50deb11f0301b99 2024-12-09T20:42:53,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/6243f728f7df4c3694bd3988d5e54a91 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6243f728f7df4c3694bd3988d5e54a91 2024-12-09T20:42:53,263 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6243f728f7df4c3694bd3988d5e54a91, entries=150, sequenceid=720, filesize=12.0 K 2024-12-09T20:42:53,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/a7c559ee41194bce9c5743683cc8dccd as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/a7c559ee41194bce9c5743683cc8dccd 2024-12-09T20:42:53,269 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/a7c559ee41194bce9c5743683cc8dccd, entries=150, sequenceid=720, filesize=12.0 K 2024-12-09T20:42:53,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/2f744a0587644584a50deb11f0301b99 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2f744a0587644584a50deb11f0301b99 2024-12-09T20:42:53,275 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2f744a0587644584a50deb11f0301b99, entries=150, sequenceid=720, filesize=12.0 K 2024-12-09T20:42:53,276 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for d1100dd05f3cf346719eddc5a46c4983 in 195ms, sequenceid=720, compaction requested=true 2024-12-09T20:42:53,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:53,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:53,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-09T20:42:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-09T20:42:53,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-09T20:42:53,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 511 msec 2024-12-09T20:42:53,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 518 msec 2024-12-09T20:42:53,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T20:42:53,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:53,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:53,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:53,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:53,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:53,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:53,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/23b789e5d0d54877a68d66fe08ace814 is 50, key is test_row_0/A:col10/1733776973227/Put/seqid=0 2024-12-09T20:42:53,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777033359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777033359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777033360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742084_1260 (size=14741) 2024-12-09T20:42:53,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777033363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777033364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T20:42:53,391 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:53,392 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-09T20:42:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T20:42:53,393 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:53,394 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:53,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:53,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777033471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777033471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777033471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777033471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777033471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,488 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/58349ea7aed9475cbfc6f29445cda437 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/58349ea7aed9475cbfc6f29445cda437 2024-12-09T20:42:53,493 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/C of d1100dd05f3cf346719eddc5a46c4983 into 58349ea7aed9475cbfc6f29445cda437(size=12.2 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:53,493 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:53,493 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/C, priority=12, startTime=1733776972979; duration=0sec 2024-12-09T20:42:53,493 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:53,493 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:42:53,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T20:42:53,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T20:42:53,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:53,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:53,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777033681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777033681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777033681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777033681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777033681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T20:42:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:53,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T20:42:53,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/23b789e5d0d54877a68d66fe08ace814 2024-12-09T20:42:53,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/b0568c967bf54fbabdf7142c5c487d14 is 50, key is test_row_0/B:col10/1733776973227/Put/seqid=0 2024-12-09T20:42:53,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742085_1261 (size=12301) 2024-12-09T20:42:53,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T20:42:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:53,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:53,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777033991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777033991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777033991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777033991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777033991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:54,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T20:42:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T20:42:54,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T20:42:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/b0568c967bf54fbabdf7142c5c487d14 2024-12-09T20:42:54,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/905254bbbd3b46ce95ad71fb6bf9df00 is 50, key is test_row_0/C:col10/1733776973227/Put/seqid=0 2024-12-09T20:42:54,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742086_1262 (size=12301) 2024-12-09T20:42:54,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T20:42:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T20:42:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777034501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:54,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777034501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:54,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777034501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777034502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:54,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777034502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:54,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:54,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T20:42:54,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/905254bbbd3b46ce95ad71fb6bf9df00 2024-12-09T20:42:54,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/23b789e5d0d54877a68d66fe08ace814 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/23b789e5d0d54877a68d66fe08ace814 2024-12-09T20:42:54,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/23b789e5d0d54877a68d66fe08ace814, entries=200, sequenceid=738, filesize=14.4 K 2024-12-09T20:42:54,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/b0568c967bf54fbabdf7142c5c487d14 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b0568c967bf54fbabdf7142c5c487d14 2024-12-09T20:42:54,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b0568c967bf54fbabdf7142c5c487d14, entries=150, sequenceid=738, filesize=12.0 K 2024-12-09T20:42:54,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/905254bbbd3b46ce95ad71fb6bf9df00 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/905254bbbd3b46ce95ad71fb6bf9df00 2024-12-09T20:42:54,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T20:42:54,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:54,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:54,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/905254bbbd3b46ce95ad71fb6bf9df00, entries=150, sequenceid=738, filesize=12.0 K 2024-12-09T20:42:54,625 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-09T20:42:54,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d1100dd05f3cf346719eddc5a46c4983 in 1282ms, sequenceid=738, compaction requested=true 2024-12-09T20:42:54,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:54,626 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:54,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:54,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:54,627 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:54,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:54,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:54,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:54,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:54,628 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:54,628 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/A is initiating minor compaction (all files) 2024-12-09T20:42:54,628 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/A in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,628 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ff98396afbf5422eb2edcab6742131d6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/661c42a43e064233bdad88970a363b80, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6243f728f7df4c3694bd3988d5e54a91, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/23b789e5d0d54877a68d66fe08ace814] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=50.6 K 2024-12-09T20:42:54,629 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:54,629 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/B is initiating minor compaction (all files) 2024-12-09T20:42:54,629 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/B in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,629 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/d7b2cb4e78dd4bebaa5f16624930f4bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3deb7337808e441b8a1d6bbdc118133d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/a7c559ee41194bce9c5743683cc8dccd, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b0568c967bf54fbabdf7142c5c487d14] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=48.3 K 2024-12-09T20:42:54,629 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff98396afbf5422eb2edcab6742131d6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733776971566 2024-12-09T20:42:54,630 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d7b2cb4e78dd4bebaa5f16624930f4bb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733776971566 2024-12-09T20:42:54,630 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 661c42a43e064233bdad88970a363b80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1733776971697 2024-12-09T20:42:54,630 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 3deb7337808e441b8a1d6bbdc118133d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1733776971697 2024-12-09T20:42:54,630 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6243f728f7df4c3694bd3988d5e54a91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733776972049 2024-12-09T20:42:54,630 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c559ee41194bce9c5743683cc8dccd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733776972049 2024-12-09T20:42:54,630 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23b789e5d0d54877a68d66fe08ace814, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1733776973218 2024-12-09T20:42:54,630 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b0568c967bf54fbabdf7142c5c487d14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1733776973227 2024-12-09T20:42:54,659 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:54,660 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/2e94f3259b43492abe7238b33c07b761 is 50, key is test_row_0/B:col10/1733776973227/Put/seqid=0 2024-12-09T20:42:54,663 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:54,664 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/1613775d3b5342f2ac9b47e48f204462 is 50, key is test_row_0/A:col10/1733776973227/Put/seqid=0 2024-12-09T20:42:54,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742087_1263 (size=12643) 2024-12-09T20:42:54,699 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/2e94f3259b43492abe7238b33c07b761 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/2e94f3259b43492abe7238b33c07b761 2024-12-09T20:42:54,705 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/B of d1100dd05f3cf346719eddc5a46c4983 into 2e94f3259b43492abe7238b33c07b761(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:54,705 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:54,705 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/B, priority=12, startTime=1733776974627; duration=0sec 2024-12-09T20:42:54,705 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:54,705 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:42:54,705 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:54,707 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37143 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:54,707 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/C is initiating minor compaction (all files) 2024-12-09T20:42:54,707 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/C in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,707 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/58349ea7aed9475cbfc6f29445cda437, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2f744a0587644584a50deb11f0301b99, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/905254bbbd3b46ce95ad71fb6bf9df00] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.3 K 2024-12-09T20:42:54,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742088_1264 (size=12643) 2024-12-09T20:42:54,708 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 58349ea7aed9475cbfc6f29445cda437, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=703, earliestPutTs=1733776971697 2024-12-09T20:42:54,709 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f744a0587644584a50deb11f0301b99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733776972049 2024-12-09T20:42:54,709 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 905254bbbd3b46ce95ad71fb6bf9df00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1733776973227 2024-12-09T20:42:54,723 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:54,724 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/38869224258f40a48dd347a640de7128 is 50, key is test_row_0/C:col10/1733776973227/Put/seqid=0 2024-12-09T20:42:54,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742089_1265 (size=12643) 2024-12-09T20:42:54,735 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/38869224258f40a48dd347a640de7128 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/38869224258f40a48dd347a640de7128 2024-12-09T20:42:54,740 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/C of d1100dd05f3cf346719eddc5a46c4983 into 38869224258f40a48dd347a640de7128(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:54,740 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:54,740 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/C, priority=13, startTime=1733776974628; duration=0sec 2024-12-09T20:42:54,741 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:54,741 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:42:54,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T20:42:54,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:54,771 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T20:42:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:54,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/9f172e5e90d545508546bb1f88854c61 is 50, key is test_row_0/A:col10/1733776973362/Put/seqid=0 2024-12-09T20:42:54,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742090_1266 (size=12301) 2024-12-09T20:42:55,118 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/1613775d3b5342f2ac9b47e48f204462 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/1613775d3b5342f2ac9b47e48f204462 2024-12-09T20:42:55,124 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/A of d1100dd05f3cf346719eddc5a46c4983 into 1613775d3b5342f2ac9b47e48f204462(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:55,124 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:55,124 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/A, priority=12, startTime=1733776974626; duration=0sec 2024-12-09T20:42:55,125 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:55,125 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:42:55,179 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/9f172e5e90d545508546bb1f88854c61 2024-12-09T20:42:55,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/6105c47f81f94c649a453bfb3c52935b is 50, key is test_row_0/B:col10/1733776973362/Put/seqid=0 2024-12-09T20:42:55,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742091_1267 (size=12301) 2024-12-09T20:42:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:55,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:55,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777035523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777035525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777035526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777035528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777035528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T20:42:55,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777035631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777035631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,637 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/6105c47f81f94c649a453bfb3c52935b 2024-12-09T20:42:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777035641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777035642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777035642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/5ebf0a242ea542b8bb4df05048c653d4 is 50, key is test_row_0/C:col10/1733776973362/Put/seqid=0 2024-12-09T20:42:55,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742092_1268 (size=12301) 2024-12-09T20:42:55,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777035841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777035842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777035851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777035851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777035851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,110 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/5ebf0a242ea542b8bb4df05048c653d4 2024-12-09T20:42:56,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/9f172e5e90d545508546bb1f88854c61 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9f172e5e90d545508546bb1f88854c61 2024-12-09T20:42:56,121 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9f172e5e90d545508546bb1f88854c61, entries=150, sequenceid=755, filesize=12.0 K 2024-12-09T20:42:56,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/6105c47f81f94c649a453bfb3c52935b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6105c47f81f94c649a453bfb3c52935b 2024-12-09T20:42:56,131 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6105c47f81f94c649a453bfb3c52935b, entries=150, sequenceid=755, filesize=12.0 K 2024-12-09T20:42:56,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/5ebf0a242ea542b8bb4df05048c653d4 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5ebf0a242ea542b8bb4df05048c653d4 2024-12-09T20:42:56,140 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5ebf0a242ea542b8bb4df05048c653d4, entries=150, sequenceid=755, filesize=12.0 K 2024-12-09T20:42:56,142 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for d1100dd05f3cf346719eddc5a46c4983 in 1370ms, sequenceid=755, compaction requested=false 2024-12-09T20:42:56,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:56,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:56,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-09T20:42:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-09T20:42:56,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-09T20:42:56,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7500 sec 2024-12-09T20:42:56,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.7600 sec 2024-12-09T20:42:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:56,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:42:56,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:56,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:56,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:56,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:56,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:56,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:42:56,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/702e76896e4f49639dc9647214e7dafc is 50, key is test_row_0/A:col10/1733776976165/Put/seqid=0 2024-12-09T20:42:56,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777036181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777036184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777036184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777036185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777036185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742093_1269 (size=12301) 2024-12-09T20:42:56,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=778 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/702e76896e4f49639dc9647214e7dafc 2024-12-09T20:42:56,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/87524fbff9a64bb88a2deb42313903c3 is 50, key is test_row_0/B:col10/1733776976165/Put/seqid=0 2024-12-09T20:42:56,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742094_1270 (size=12301) 2024-12-09T20:42:56,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=778 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/87524fbff9a64bb88a2deb42313903c3 2024-12-09T20:42:56,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/c5ba6c8301cc4a8291b43ee16d416d94 is 50, key is test_row_0/C:col10/1733776976165/Put/seqid=0 2024-12-09T20:42:56,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742095_1271 (size=12301) 2024-12-09T20:42:56,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777036291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777036291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777036301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777036301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777036301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777036501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777036501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777036511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777036511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777036512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=778 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/c5ba6c8301cc4a8291b43ee16d416d94 2024-12-09T20:42:56,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/702e76896e4f49639dc9647214e7dafc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/702e76896e4f49639dc9647214e7dafc 2024-12-09T20:42:56,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/702e76896e4f49639dc9647214e7dafc, entries=150, sequenceid=778, filesize=12.0 K 2024-12-09T20:42:56,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/87524fbff9a64bb88a2deb42313903c3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/87524fbff9a64bb88a2deb42313903c3 2024-12-09T20:42:56,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/87524fbff9a64bb88a2deb42313903c3, entries=150, sequenceid=778, filesize=12.0 K 2024-12-09T20:42:56,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/c5ba6c8301cc4a8291b43ee16d416d94 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c5ba6c8301cc4a8291b43ee16d416d94 2024-12-09T20:42:56,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c5ba6c8301cc4a8291b43ee16d416d94, entries=150, sequenceid=778, filesize=12.0 K 2024-12-09T20:42:56,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for d1100dd05f3cf346719eddc5a46c4983 in 545ms, sequenceid=778, compaction requested=true 2024-12-09T20:42:56,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:56,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:56,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:56,710 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:56,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:56,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:56,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:56,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:42:56,710 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:56,711 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:56,711 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/A is initiating minor compaction (all files) 2024-12-09T20:42:56,711 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:56,711 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/B is initiating minor compaction (all files) 2024-12-09T20:42:56,712 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/A in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:56,712 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/B in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:56,712 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/1613775d3b5342f2ac9b47e48f204462, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9f172e5e90d545508546bb1f88854c61, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/702e76896e4f49639dc9647214e7dafc] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.4 K 2024-12-09T20:42:56,712 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/2e94f3259b43492abe7238b33c07b761, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6105c47f81f94c649a453bfb3c52935b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/87524fbff9a64bb88a2deb42313903c3] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.4 K 2024-12-09T20:42:56,712 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e94f3259b43492abe7238b33c07b761, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1733776973227 2024-12-09T20:42:56,712 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1613775d3b5342f2ac9b47e48f204462, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1733776973227 2024-12-09T20:42:56,712 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6105c47f81f94c649a453bfb3c52935b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=755, earliestPutTs=1733776973356 2024-12-09T20:42:56,712 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f172e5e90d545508546bb1f88854c61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=755, earliestPutTs=1733776973356 2024-12-09T20:42:56,713 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702e76896e4f49639dc9647214e7dafc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=778, earliestPutTs=1733776975526 2024-12-09T20:42:56,713 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 87524fbff9a64bb88a2deb42313903c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=778, earliestPutTs=1733776975526 2024-12-09T20:42:56,739 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:56,740 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4ca462cb7556418bae74d158d112ce5c is 50, key is test_row_0/B:col10/1733776976165/Put/seqid=0 2024-12-09T20:42:56,751 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#A#compaction#229 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:56,751 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/b14a4cedc870456588f68abe35bff62f is 50, key is test_row_0/A:col10/1733776976165/Put/seqid=0 2024-12-09T20:42:56,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742096_1272 (size=12745) 2024-12-09T20:42:56,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742097_1273 (size=12745) 2024-12-09T20:42:56,792 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/b14a4cedc870456588f68abe35bff62f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/b14a4cedc870456588f68abe35bff62f 2024-12-09T20:42:56,798 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/A of d1100dd05f3cf346719eddc5a46c4983 into b14a4cedc870456588f68abe35bff62f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:56,798 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:56,798 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/A, priority=13, startTime=1733776976710; duration=0sec 2024-12-09T20:42:56,798 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:56,798 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:42:56,799 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:42:56,801 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:42:56,801 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/C is initiating minor compaction (all files) 2024-12-09T20:42:56,801 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/C in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:56,801 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/38869224258f40a48dd347a640de7128, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5ebf0a242ea542b8bb4df05048c653d4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c5ba6c8301cc4a8291b43ee16d416d94] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.4 K 2024-12-09T20:42:56,801 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38869224258f40a48dd347a640de7128, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1733776973227 2024-12-09T20:42:56,802 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ebf0a242ea542b8bb4df05048c653d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=755, earliestPutTs=1733776973356 2024-12-09T20:42:56,802 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ba6c8301cc4a8291b43ee16d416d94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=778, earliestPutTs=1733776975526 2024-12-09T20:42:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:56,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:42:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:56,816 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:56,817 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/1f50de20262249a89ba3fd2f4467fff0 is 50, key is test_row_0/C:col10/1733776976165/Put/seqid=0 2024-12-09T20:42:56,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/3eeb52afbbdb47aa9c103787b2f12307 is 50, key is test_row_0/A:col10/1733776976183/Put/seqid=0 2024-12-09T20:42:56,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777036836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777036836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777036836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777036839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777036841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742098_1274 (size=12745) 2024-12-09T20:42:56,850 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/1f50de20262249a89ba3fd2f4467fff0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1f50de20262249a89ba3fd2f4467fff0 2024-12-09T20:42:56,857 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/C of d1100dd05f3cf346719eddc5a46c4983 into 1f50de20262249a89ba3fd2f4467fff0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:56,857 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:56,857 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/C, priority=13, startTime=1733776976710; duration=0sec 2024-12-09T20:42:56,857 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:56,857 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:42:56,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742099_1275 (size=12301) 2024-12-09T20:42:56,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777036952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777036952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777036952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777036952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777036952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:56,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777037161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777037162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777037162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777037162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777037162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,184 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4ca462cb7556418bae74d158d112ce5c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4ca462cb7556418bae74d158d112ce5c 2024-12-09T20:42:57,190 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/B of d1100dd05f3cf346719eddc5a46c4983 into 4ca462cb7556418bae74d158d112ce5c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:57,190 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:57,190 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/B, priority=13, startTime=1733776976710; duration=0sec 2024-12-09T20:42:57,190 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:57,191 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:42:57,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=791 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/3eeb52afbbdb47aa9c103787b2f12307 2024-12-09T20:42:57,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/77abc15c7e2244a89f22a21958f1bea5 is 50, key is test_row_0/B:col10/1733776976183/Put/seqid=0 2024-12-09T20:42:57,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742100_1276 (size=12301) 2024-12-09T20:42:57,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777037471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777037471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777037471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777037472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777037472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T20:42:57,562 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:57,562 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-09T20:42:57,623 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T20:42:57,624 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:57,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T20:42:57,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=791 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/77abc15c7e2244a89f22a21958f1bea5 2024-12-09T20:42:57,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4cdcf0c77b094a0ba3dcdbc8fa07f137 is 50, key is test_row_0/C:col10/1733776976183/Put/seqid=0 2024-12-09T20:42:57,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T20:42:57,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:57,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:57,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:57,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:57,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:57,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742101_1277 (size=12301) 2024-12-09T20:42:57,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=791 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4cdcf0c77b094a0ba3dcdbc8fa07f137 2024-12-09T20:42:57,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/3eeb52afbbdb47aa9c103787b2f12307 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/3eeb52afbbdb47aa9c103787b2f12307 2024-12-09T20:42:57,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/3eeb52afbbdb47aa9c103787b2f12307, entries=150, sequenceid=791, filesize=12.0 K 2024-12-09T20:42:57,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/77abc15c7e2244a89f22a21958f1bea5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/77abc15c7e2244a89f22a21958f1bea5 2024-12-09T20:42:57,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/77abc15c7e2244a89f22a21958f1bea5, entries=150, sequenceid=791, filesize=12.0 K 2024-12-09T20:42:57,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4cdcf0c77b094a0ba3dcdbc8fa07f137 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4cdcf0c77b094a0ba3dcdbc8fa07f137 2024-12-09T20:42:57,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4cdcf0c77b094a0ba3dcdbc8fa07f137, entries=150, sequenceid=791, filesize=12.0 K 2024-12-09T20:42:57,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d1100dd05f3cf346719eddc5a46c4983 in 1038ms, sequenceid=791, compaction requested=false 2024-12-09T20:42:57,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:57,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T20:42:57,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:57,935 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:42:57,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:57,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:57,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:57,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:57,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:57,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:57,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/22f059a360834ac0b65cc64acc2d1714 is 50, key is test_row_0/A:col10/1733776976832/Put/seqid=0 2024-12-09T20:42:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T20:42:57,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742102_1278 (size=12301) 2024-12-09T20:42:57,953 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=813 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/22f059a360834ac0b65cc64acc2d1714 2024-12-09T20:42:57,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/dbf12eda62974e5290a17ada8f46d7a9 is 50, key is test_row_0/B:col10/1733776976832/Put/seqid=0 2024-12-09T20:42:57,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:58,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777038001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777038002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777038001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742103_1279 (size=12301) 2024-12-09T20:42:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777038010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777038014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777038121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777038122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777038122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777038122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777038122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T20:42:58,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777038332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777038332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777038332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777038332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777038332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,413 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=813 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/dbf12eda62974e5290a17ada8f46d7a9 2024-12-09T20:42:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4ee8c3840c084f3fa73e4da9d7ab26b3 is 50, key is test_row_0/C:col10/1733776976832/Put/seqid=0 2024-12-09T20:42:58,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742104_1280 (size=12301) 2024-12-09T20:42:58,473 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=813 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4ee8c3840c084f3fa73e4da9d7ab26b3 2024-12-09T20:42:58,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/22f059a360834ac0b65cc64acc2d1714 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/22f059a360834ac0b65cc64acc2d1714 2024-12-09T20:42:58,483 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/22f059a360834ac0b65cc64acc2d1714, entries=150, sequenceid=813, filesize=12.0 K 2024-12-09T20:42:58,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/dbf12eda62974e5290a17ada8f46d7a9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dbf12eda62974e5290a17ada8f46d7a9 2024-12-09T20:42:58,489 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dbf12eda62974e5290a17ada8f46d7a9, entries=150, sequenceid=813, filesize=12.0 K 2024-12-09T20:42:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4ee8c3840c084f3fa73e4da9d7ab26b3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4ee8c3840c084f3fa73e4da9d7ab26b3 2024-12-09T20:42:58,495 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4ee8c3840c084f3fa73e4da9d7ab26b3, entries=150, sequenceid=813, filesize=12.0 K 2024-12-09T20:42:58,497 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d1100dd05f3cf346719eddc5a46c4983 in 562ms, sequenceid=813, compaction requested=true 2024-12-09T20:42:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:58,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-09T20:42:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-09T20:42:58,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-09T20:42:58,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 876 msec 2024-12-09T20:42:58,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 941 msec 2024-12-09T20:42:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:58,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:42:58,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:58,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:58,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:58,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:58,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:58,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:58,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/8eb71ad51d144a7cb2419362c1038c47 is 50, key is test_row_0/A:col10/1733776978007/Put/seqid=0 2024-12-09T20:42:58,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777038668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777038671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777038672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777038672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777038678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742105_1281 (size=12301) 2024-12-09T20:42:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T20:42:58,762 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:42:58,763 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:42:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-09T20:42:58,765 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:42:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T20:42:58,766 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:42:58,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:42:58,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777038781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777038781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777038781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777038781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777038791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T20:42:58,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T20:42:58,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:58,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:58,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:58,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:58,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:58,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777038991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777038991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777038991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777038991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:58,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777039001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T20:42:59,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:59,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T20:42:59,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=827 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/8eb71ad51d144a7cb2419362c1038c47 2024-12-09T20:42:59,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/38e377a3676e4f76aec28bdb1a88b5fe is 50, key is test_row_0/B:col10/1733776978007/Put/seqid=0 2024-12-09T20:42:59,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742106_1282 (size=12301) 2024-12-09T20:42:59,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T20:42:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777039301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777039302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777039302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777039302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777039311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T20:42:59,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:59,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T20:42:59,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T20:42:59,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:42:59,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=827 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/38e377a3676e4f76aec28bdb1a88b5fe 2024-12-09T20:42:59,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/39c344a15647489eabab6c7221f801dd is 50, key is test_row_0/C:col10/1733776978007/Put/seqid=0 2024-12-09T20:42:59,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742107_1283 (size=12301) 2024-12-09T20:42:59,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=827 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/39c344a15647489eabab6c7221f801dd 2024-12-09T20:42:59,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/8eb71ad51d144a7cb2419362c1038c47 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/8eb71ad51d144a7cb2419362c1038c47 2024-12-09T20:42:59,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/8eb71ad51d144a7cb2419362c1038c47, entries=150, sequenceid=827, filesize=12.0 K 2024-12-09T20:42:59,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/38e377a3676e4f76aec28bdb1a88b5fe as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/38e377a3676e4f76aec28bdb1a88b5fe 2024-12-09T20:42:59,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/38e377a3676e4f76aec28bdb1a88b5fe, entries=150, sequenceid=827, filesize=12.0 K 2024-12-09T20:42:59,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/39c344a15647489eabab6c7221f801dd as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/39c344a15647489eabab6c7221f801dd 2024-12-09T20:42:59,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/39c344a15647489eabab6c7221f801dd, entries=150, sequenceid=827, filesize=12.0 K 2024-12-09T20:42:59,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d1100dd05f3cf346719eddc5a46c4983 in 988ms, sequenceid=827, compaction requested=true 2024-12-09T20:42:59,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:59,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:42:59,635 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:59,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:59,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:42:59,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:59,635 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:59,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:42:59,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:59,637 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:59,637 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:59,637 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/B is initiating minor compaction (all files) 2024-12-09T20:42:59,637 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/A is initiating minor compaction (all files) 2024-12-09T20:42:59,637 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/B in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,637 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/A in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,637 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/b14a4cedc870456588f68abe35bff62f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/3eeb52afbbdb47aa9c103787b2f12307, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/22f059a360834ac0b65cc64acc2d1714, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/8eb71ad51d144a7cb2419362c1038c47] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=48.5 K 2024-12-09T20:42:59,637 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4ca462cb7556418bae74d158d112ce5c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/77abc15c7e2244a89f22a21958f1bea5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dbf12eda62974e5290a17ada8f46d7a9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/38e377a3676e4f76aec28bdb1a88b5fe] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=48.5 K 2024-12-09T20:42:59,638 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca462cb7556418bae74d158d112ce5c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=778, earliestPutTs=1733776975526 2024-12-09T20:42:59,638 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting b14a4cedc870456588f68abe35bff62f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=778, earliestPutTs=1733776975526 2024-12-09T20:42:59,638 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eeb52afbbdb47aa9c103787b2f12307, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=791, earliestPutTs=1733776976179 2024-12-09T20:42:59,638 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 77abc15c7e2244a89f22a21958f1bea5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=791, earliestPutTs=1733776976179 2024-12-09T20:42:59,639 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf12eda62974e5290a17ada8f46d7a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=813, earliestPutTs=1733776976825 2024-12-09T20:42:59,639 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22f059a360834ac0b65cc64acc2d1714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=813, earliestPutTs=1733776976825 2024-12-09T20:42:59,639 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 38e377a3676e4f76aec28bdb1a88b5fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=827, earliestPutTs=1733776978007 2024-12-09T20:42:59,639 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eb71ad51d144a7cb2419362c1038c47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=827, earliestPutTs=1733776978007 2024-12-09T20:42:59,663 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#A#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:59,663 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/64d1a2760c114d5e9c28da2aab02918a is 50, key is test_row_0/A:col10/1733776978007/Put/seqid=0 2024-12-09T20:42:59,665 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:59,666 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/18763f96573f440d9f3cc0d34e4da3d2 is 50, key is test_row_0/B:col10/1733776978007/Put/seqid=0 2024-12-09T20:42:59,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742108_1284 (size=12881) 2024-12-09T20:42:59,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742109_1285 (size=12881) 2024-12-09T20:42:59,674 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/64d1a2760c114d5e9c28da2aab02918a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/64d1a2760c114d5e9c28da2aab02918a 2024-12-09T20:42:59,678 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/18763f96573f440d9f3cc0d34e4da3d2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/18763f96573f440d9f3cc0d34e4da3d2 2024-12-09T20:42:59,681 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/A of d1100dd05f3cf346719eddc5a46c4983 into 64d1a2760c114d5e9c28da2aab02918a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:59,681 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:59,681 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/A, priority=12, startTime=1733776979635; duration=0sec 2024-12-09T20:42:59,681 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:42:59,681 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:42:59,681 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:42:59,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:42:59,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/C is initiating minor compaction (all files) 2024-12-09T20:42:59,683 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/C in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,684 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1f50de20262249a89ba3fd2f4467fff0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4cdcf0c77b094a0ba3dcdbc8fa07f137, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4ee8c3840c084f3fa73e4da9d7ab26b3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/39c344a15647489eabab6c7221f801dd] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=48.5 K 2024-12-09T20:42:59,684 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/B of d1100dd05f3cf346719eddc5a46c4983 into 18763f96573f440d9f3cc0d34e4da3d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:42:59,684 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:42:59,684 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/B, priority=12, startTime=1733776979635; duration=0sec 2024-12-09T20:42:59,684 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f50de20262249a89ba3fd2f4467fff0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=778, earliestPutTs=1733776975526 2024-12-09T20:42:59,684 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:42:59,684 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:42:59,685 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cdcf0c77b094a0ba3dcdbc8fa07f137, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=791, earliestPutTs=1733776976179 2024-12-09T20:42:59,685 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ee8c3840c084f3fa73e4da9d7ab26b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=813, earliestPutTs=1733776976825 2024-12-09T20:42:59,685 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39c344a15647489eabab6c7221f801dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=827, earliestPutTs=1733776978007 2024-12-09T20:42:59,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T20:42:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:42:59,689 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T20:42:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:42:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:42:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:42:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:42:59,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/17343bf622ab473db76f26a7b5339418 is 50, key is test_row_0/A:col10/1733776978669/Put/seqid=0 2024-12-09T20:42:59,722 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:42:59,722 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/2c34375c89f94d36b1090b51fd0b551f is 50, key is test_row_0/C:col10/1733776978007/Put/seqid=0 2024-12-09T20:42:59,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742110_1286 (size=12301) 2024-12-09T20:42:59,744 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=847 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/17343bf622ab473db76f26a7b5339418 2024-12-09T20:42:59,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742111_1287 (size=12881) 2024-12-09T20:42:59,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3d55f58574c44ca683e51520d6faeca2 is 50, key is test_row_0/B:col10/1733776978669/Put/seqid=0 2024-12-09T20:42:59,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742112_1288 (size=12301) 2024-12-09T20:42:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:42:59,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:42:59,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777039824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777039825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777039826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777039827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777039829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T20:42:59,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777039941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777039941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777039942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777039942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:42:59,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:42:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777039942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:42:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:42:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,149 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/2c34375c89f94d36b1090b51fd0b551f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2c34375c89f94d36b1090b51fd0b551f 2024-12-09T20:43:00,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777040151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777040152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777040152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777040152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777040152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,156 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/C of d1100dd05f3cf346719eddc5a46c4983 into 2c34375c89f94d36b1090b51fd0b551f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:00,156 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:00,156 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/C, priority=12, startTime=1733776979635; duration=0sec 2024-12-09T20:43:00,156 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:00,156 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:43:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,207 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=847 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3d55f58574c44ca683e51520d6faeca2 2024-12-09T20:43:00,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/1321884cce7944128be69f75319c6637 is 50, key is test_row_0/C:col10/1733776978669/Put/seqid=0 2024-12-09T20:43:00,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742113_1289 (size=12301) 2024-12-09T20:43:00,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777040461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777040462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777040462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777040464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777040464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:00,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:00,626 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=847 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/1321884cce7944128be69f75319c6637 2024-12-09T20:43:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/17343bf622ab473db76f26a7b5339418 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17343bf622ab473db76f26a7b5339418 2024-12-09T20:43:00,645 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17343bf622ab473db76f26a7b5339418, entries=150, sequenceid=847, filesize=12.0 K 2024-12-09T20:43:00,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3d55f58574c44ca683e51520d6faeca2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3d55f58574c44ca683e51520d6faeca2 2024-12-09T20:43:00,649 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3d55f58574c44ca683e51520d6faeca2, entries=150, sequenceid=847, filesize=12.0 K 2024-12-09T20:43:00,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/1321884cce7944128be69f75319c6637 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1321884cce7944128be69f75319c6637 2024-12-09T20:43:00,653 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1321884cce7944128be69f75319c6637, entries=150, sequenceid=847, filesize=12.0 K 2024-12-09T20:43:00,653 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d1100dd05f3cf346719eddc5a46c4983 in 964ms, sequenceid=847, compaction requested=false 2024-12-09T20:43:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-09T20:43:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-09T20:43:00,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-09T20:43:00,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8890 sec 2024-12-09T20:43:00,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.8940 sec 2024-12-09T20:43:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T20:43:00,911 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:00,912 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-09T20:43:00,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:00,915 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:00,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T20:43:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:00,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T20:43:00,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:00,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:00,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:00,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:00,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:00,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:00,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/18a84a9dab9041fb9044f391a87e6ed9 is 50, key is test_row_0/A:col10/1733776980972/Put/seqid=0 2024-12-09T20:43:00,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742114_1290 (size=12301) 2024-12-09T20:43:01,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777040994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777040998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777040998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777041001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777041002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T20:43:01,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T20:43:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777041111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777041112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777041111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777041112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777041112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T20:43:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T20:43:01,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777041321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777041321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777041321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777041321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777041322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T20:43:01,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:01,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=865 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/18a84a9dab9041fb9044f391a87e6ed9 2024-12-09T20:43:01,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/f4bcce0262824f6c859bad0c10c4d1a3 is 50, key is test_row_0/B:col10/1733776980972/Put/seqid=0 2024-12-09T20:43:01,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742115_1291 (size=12301) 2024-12-09T20:43:01,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=865 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/f4bcce0262824f6c859bad0c10c4d1a3 2024-12-09T20:43:01,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/8f82f06f522b47228873b0aade6c0b7d is 50, key is test_row_0/C:col10/1733776980972/Put/seqid=0 2024-12-09T20:43:01,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742116_1292 (size=12301) 2024-12-09T20:43:01,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T20:43:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T20:43:01,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777041631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777041631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777041632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777041632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777041632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:01,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T20:43:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:01,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:01,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=865 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/8f82f06f522b47228873b0aade6c0b7d 2024-12-09T20:43:01,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/18a84a9dab9041fb9044f391a87e6ed9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/18a84a9dab9041fb9044f391a87e6ed9 2024-12-09T20:43:01,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/18a84a9dab9041fb9044f391a87e6ed9, entries=150, sequenceid=865, filesize=12.0 K 2024-12-09T20:43:01,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/f4bcce0262824f6c859bad0c10c4d1a3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/f4bcce0262824f6c859bad0c10c4d1a3 2024-12-09T20:43:01,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/f4bcce0262824f6c859bad0c10c4d1a3, entries=150, sequenceid=865, filesize=12.0 K 2024-12-09T20:43:01,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/8f82f06f522b47228873b0aade6c0b7d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8f82f06f522b47228873b0aade6c0b7d 2024-12-09T20:43:01,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8f82f06f522b47228873b0aade6c0b7d, entries=150, sequenceid=865, filesize=12.0 K 2024-12-09T20:43:01,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d1100dd05f3cf346719eddc5a46c4983 in 852ms, sequenceid=865, compaction requested=true 2024-12-09T20:43:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:01,827 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:01,827 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:01,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:01,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:01,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/A is initiating minor compaction (all files) 2024-12-09T20:43:01,828 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/A in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,828 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:01,828 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/B is initiating minor compaction (all files) 2024-12-09T20:43:01,828 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/B in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,828 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/64d1a2760c114d5e9c28da2aab02918a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17343bf622ab473db76f26a7b5339418, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/18a84a9dab9041fb9044f391a87e6ed9] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.6 K 2024-12-09T20:43:01,828 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/18763f96573f440d9f3cc0d34e4da3d2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3d55f58574c44ca683e51520d6faeca2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/f4bcce0262824f6c859bad0c10c4d1a3] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.6 K 2024-12-09T20:43:01,828 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 18763f96573f440d9f3cc0d34e4da3d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=827, earliestPutTs=1733776978007 2024-12-09T20:43:01,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d1a2760c114d5e9c28da2aab02918a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=827, earliestPutTs=1733776978007 2024-12-09T20:43:01,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17343bf622ab473db76f26a7b5339418, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=847, earliestPutTs=1733776978669 2024-12-09T20:43:01,828 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d55f58574c44ca683e51520d6faeca2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=847, earliestPutTs=1733776978669 2024-12-09T20:43:01,829 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f4bcce0262824f6c859bad0c10c4d1a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=865, earliestPutTs=1733776979823 2024-12-09T20:43:01,829 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18a84a9dab9041fb9044f391a87e6ed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=865, earliestPutTs=1733776979823 2024-12-09T20:43:01,835 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#A#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:01,835 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:01,835 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/d7aa183db3df4e0fa00279e9b2ac027e is 50, key is test_row_0/A:col10/1733776980972/Put/seqid=0 2024-12-09T20:43:01,835 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3afac610941142089d0a8610c355c4df is 50, key is test_row_0/B:col10/1733776980972/Put/seqid=0 2024-12-09T20:43:01,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T20:43:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,838 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T20:43:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:01,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742117_1293 (size=12983) 2024-12-09T20:43:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742118_1294 (size=12983) 2024-12-09T20:43:01,870 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3afac610941142089d0a8610c355c4df as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3afac610941142089d0a8610c355c4df 2024-12-09T20:43:01,879 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/B of d1100dd05f3cf346719eddc5a46c4983 into 3afac610941142089d0a8610c355c4df(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:01,879 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:01,880 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/B, priority=13, startTime=1733776981827; duration=0sec 2024-12-09T20:43:01,880 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:01,880 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:43:01,880 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:01,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/f89c29924c15457c90a97e5aeb1eae47 is 50, key is test_row_0/A:col10/1733776980999/Put/seqid=0 2024-12-09T20:43:01,883 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:01,884 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/C is initiating minor compaction (all files) 2024-12-09T20:43:01,884 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/C in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:01,884 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2c34375c89f94d36b1090b51fd0b551f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1321884cce7944128be69f75319c6637, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8f82f06f522b47228873b0aade6c0b7d] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.6 K 2024-12-09T20:43:01,885 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c34375c89f94d36b1090b51fd0b551f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=827, earliestPutTs=1733776978007 2024-12-09T20:43:01,885 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 1321884cce7944128be69f75319c6637, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=847, earliestPutTs=1733776978669 2024-12-09T20:43:01,885 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f82f06f522b47228873b0aade6c0b7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=865, earliestPutTs=1733776979823 2024-12-09T20:43:01,895 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:01,896 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/03dd75a61502430f9ae7afff1f122944 is 50, key is test_row_0/C:col10/1733776980972/Put/seqid=0 2024-12-09T20:43:01,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742119_1295 (size=12301) 2024-12-09T20:43:01,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742120_1296 (size=12983) 2024-12-09T20:43:01,936 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/03dd75a61502430f9ae7afff1f122944 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/03dd75a61502430f9ae7afff1f122944 2024-12-09T20:43:01,942 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/C of d1100dd05f3cf346719eddc5a46c4983 into 03dd75a61502430f9ae7afff1f122944(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:01,942 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:01,942 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/C, priority=13, startTime=1733776981827; duration=0sec 2024-12-09T20:43:01,942 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:01,942 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:43:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T20:43:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:02,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:02,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777042154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777042157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777042158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777042159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777042159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,250 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/d7aa183db3df4e0fa00279e9b2ac027e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/d7aa183db3df4e0fa00279e9b2ac027e 2024-12-09T20:43:02,253 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/A of d1100dd05f3cf346719eddc5a46c4983 into d7aa183db3df4e0fa00279e9b2ac027e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:02,253 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:02,253 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/A, priority=13, startTime=1733776981827; duration=0sec 2024-12-09T20:43:02,253 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:02,253 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:43:02,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777042261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777042271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777042271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777042272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777042272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,323 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=880 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/f89c29924c15457c90a97e5aeb1eae47 2024-12-09T20:43:02,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/b81d95d4c4d44243822572624081f890 is 50, key is test_row_0/B:col10/1733776980999/Put/seqid=0 2024-12-09T20:43:02,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742121_1297 (size=12301) 2024-12-09T20:43:02,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777042471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777042481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777042482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777042482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777042482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,742 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=880 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/b81d95d4c4d44243822572624081f890 2024-12-09T20:43:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/dae4599408874a60ab3ded936c00dd43 is 50, key is test_row_0/C:col10/1733776980999/Put/seqid=0 2024-12-09T20:43:02,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742122_1298 (size=12301) 2024-12-09T20:43:02,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777042781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777042791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777042791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777042791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777042791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T20:43:03,171 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=880 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/dae4599408874a60ab3ded936c00dd43 2024-12-09T20:43:03,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/f89c29924c15457c90a97e5aeb1eae47 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/f89c29924c15457c90a97e5aeb1eae47 2024-12-09T20:43:03,179 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/f89c29924c15457c90a97e5aeb1eae47, entries=150, sequenceid=880, filesize=12.0 K 2024-12-09T20:43:03,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/b81d95d4c4d44243822572624081f890 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b81d95d4c4d44243822572624081f890 2024-12-09T20:43:03,183 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b81d95d4c4d44243822572624081f890, entries=150, sequenceid=880, filesize=12.0 K 2024-12-09T20:43:03,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/dae4599408874a60ab3ded936c00dd43 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/dae4599408874a60ab3ded936c00dd43 2024-12-09T20:43:03,187 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/dae4599408874a60ab3ded936c00dd43, entries=150, sequenceid=880, filesize=12.0 K 2024-12-09T20:43:03,188 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for d1100dd05f3cf346719eddc5a46c4983 in 1349ms, sequenceid=880, compaction requested=false 2024-12-09T20:43:03,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:03,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:03,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-09T20:43:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-09T20:43:03,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-09T20:43:03,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2740 sec 2024-12-09T20:43:03,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2790 sec 2024-12-09T20:43:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:03,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:43:03,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:03,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:03,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:03,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:03,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:03,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:03,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/6e4103f21bce4d10832cef162939f885 is 50, key is test_row_0/A:col10/1733776982157/Put/seqid=0 2024-12-09T20:43:03,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742123_1299 (size=14741) 2024-12-09T20:43:03,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777043311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777043312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777043313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777043312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777043313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777043421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777043421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777043421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777043421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777043421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777043631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777043631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777043631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777043631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777043631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=902 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/6e4103f21bce4d10832cef162939f885 2024-12-09T20:43:03,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/8abbd335420e427390231d5d7ece7010 is 50, key is test_row_0/B:col10/1733776982157/Put/seqid=0 2024-12-09T20:43:03,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742124_1300 (size=12301) 2024-12-09T20:43:03,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777043941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777043941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777043941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777043941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777043941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:04,113 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T20:43:04,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=902 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/8abbd335420e427390231d5d7ece7010 2024-12-09T20:43:04,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/0a51969cb4794e379c72c1f9f78c9085 is 50, key is test_row_0/C:col10/1733776982157/Put/seqid=0 2024-12-09T20:43:04,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742125_1301 (size=12301) 2024-12-09T20:43:04,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:04,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777044451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777044452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:04,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777044452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777044452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777044452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:04,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=902 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/0a51969cb4794e379c72c1f9f78c9085 2024-12-09T20:43:04,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/6e4103f21bce4d10832cef162939f885 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6e4103f21bce4d10832cef162939f885 2024-12-09T20:43:04,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6e4103f21bce4d10832cef162939f885, entries=200, sequenceid=902, filesize=14.4 K 2024-12-09T20:43:04,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/8abbd335420e427390231d5d7ece7010 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/8abbd335420e427390231d5d7ece7010 2024-12-09T20:43:04,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/8abbd335420e427390231d5d7ece7010, entries=150, sequenceid=902, filesize=12.0 K 2024-12-09T20:43:04,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/0a51969cb4794e379c72c1f9f78c9085 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/0a51969cb4794e379c72c1f9f78c9085 2024-12-09T20:43:04,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/0a51969cb4794e379c72c1f9f78c9085, entries=150, sequenceid=902, filesize=12.0 K 2024-12-09T20:43:04,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for d1100dd05f3cf346719eddc5a46c4983 in 1293ms, sequenceid=902, compaction requested=true 2024-12-09T20:43:04,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:04,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:04,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:04,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:04,586 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:04,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:04,587 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:04,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:04,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:04,588 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:04,588 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/B is initiating minor compaction (all files) 2024-12-09T20:43:04,588 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/B in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:04,588 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3afac610941142089d0a8610c355c4df, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b81d95d4c4d44243822572624081f890, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/8abbd335420e427390231d5d7ece7010] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.7 K 2024-12-09T20:43:04,588 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:04,588 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/A is initiating minor compaction (all files) 2024-12-09T20:43:04,588 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/A in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:04,589 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/d7aa183db3df4e0fa00279e9b2ac027e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/f89c29924c15457c90a97e5aeb1eae47, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6e4103f21bce4d10832cef162939f885] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=39.1 K 2024-12-09T20:43:04,589 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 3afac610941142089d0a8610c355c4df, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=865, earliestPutTs=1733776979823 2024-12-09T20:43:04,589 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7aa183db3df4e0fa00279e9b2ac027e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=865, earliestPutTs=1733776979823 2024-12-09T20:43:04,589 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b81d95d4c4d44243822572624081f890, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=880, earliestPutTs=1733776980992 2024-12-09T20:43:04,589 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting f89c29924c15457c90a97e5aeb1eae47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=880, earliestPutTs=1733776980992 2024-12-09T20:43:04,589 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e4103f21bce4d10832cef162939f885, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=902, earliestPutTs=1733776982155 2024-12-09T20:43:04,589 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8abbd335420e427390231d5d7ece7010, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=902, earliestPutTs=1733776982157 2024-12-09T20:43:04,599 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:04,600 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4a928038ecf045d980697dcea335af1c is 50, key is test_row_0/B:col10/1733776982157/Put/seqid=0 2024-12-09T20:43:04,600 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#A#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:04,600 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/ca583fc9d73c46859981baf60fda1d68 is 50, key is test_row_0/A:col10/1733776982157/Put/seqid=0 2024-12-09T20:43:04,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742127_1303 (size=13085) 2024-12-09T20:43:04,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742126_1302 (size=13085) 2024-12-09T20:43:05,028 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4a928038ecf045d980697dcea335af1c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4a928038ecf045d980697dcea335af1c 2024-12-09T20:43:05,036 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/ca583fc9d73c46859981baf60fda1d68 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ca583fc9d73c46859981baf60fda1d68 2024-12-09T20:43:05,036 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/B of d1100dd05f3cf346719eddc5a46c4983 into 4a928038ecf045d980697dcea335af1c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:05,036 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:05,037 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/B, priority=13, startTime=1733776984586; duration=0sec 2024-12-09T20:43:05,037 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:05,037 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:43:05,037 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:05,044 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:05,044 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/C is initiating minor compaction (all files) 2024-12-09T20:43:05,045 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/C in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:05,045 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/03dd75a61502430f9ae7afff1f122944, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/dae4599408874a60ab3ded936c00dd43, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/0a51969cb4794e379c72c1f9f78c9085] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.7 K 2024-12-09T20:43:05,047 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/A of d1100dd05f3cf346719eddc5a46c4983 into ca583fc9d73c46859981baf60fda1d68(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:05,047 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:05,049 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/A, priority=13, startTime=1733776984586; duration=0sec 2024-12-09T20:43:05,049 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 03dd75a61502430f9ae7afff1f122944, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=865, earliestPutTs=1733776979823 2024-12-09T20:43:05,049 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:05,049 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:43:05,050 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting dae4599408874a60ab3ded936c00dd43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=880, earliestPutTs=1733776980992 2024-12-09T20:43:05,050 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a51969cb4794e379c72c1f9f78c9085, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=902, earliestPutTs=1733776982157 2024-12-09T20:43:05,071 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:05,072 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/216517747f5543ce9f895ca2eda8af9c is 50, key is test_row_0/C:col10/1733776982157/Put/seqid=0 2024-12-09T20:43:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T20:43:05,092 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:05,092 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-09T20:43:05,095 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:05,097 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:05,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T20:43:05,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742128_1304 (size=13085) 2024-12-09T20:43:05,135 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/216517747f5543ce9f895ca2eda8af9c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/216517747f5543ce9f895ca2eda8af9c 2024-12-09T20:43:05,140 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/C of d1100dd05f3cf346719eddc5a46c4983 into 216517747f5543ce9f895ca2eda8af9c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:05,140 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:05,140 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/C, priority=13, startTime=1733776984587; duration=0sec 2024-12-09T20:43:05,140 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:05,140 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:43:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T20:43:05,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T20:43:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:05,251 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T20:43:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:05,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/aee3be9e5710489e87bde828af3486a2 is 50, key is test_row_0/A:col10/1733776983312/Put/seqid=0 2024-12-09T20:43:05,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742129_1305 (size=12301) 2024-12-09T20:43:05,293 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=919 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/aee3be9e5710489e87bde828af3486a2 2024-12-09T20:43:05,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4547e5db7f324661856d3bff55d9684e is 50, key is test_row_0/B:col10/1733776983312/Put/seqid=0 2024-12-09T20:43:05,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742130_1306 (size=12301) 2024-12-09T20:43:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T20:43:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:05,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:05,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777045473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777045474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777045475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777045476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777045477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777045583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777045583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777045584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777045584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777045591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,714 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=919 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4547e5db7f324661856d3bff55d9684e 2024-12-09T20:43:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T20:43:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/bf9125ac20a54ca6a4ea2e9d0d29398e is 50, key is test_row_0/C:col10/1733776983312/Put/seqid=0 2024-12-09T20:43:05,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742131_1307 (size=12301) 2024-12-09T20:43:05,729 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=919 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/bf9125ac20a54ca6a4ea2e9d0d29398e 2024-12-09T20:43:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/aee3be9e5710489e87bde828af3486a2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/aee3be9e5710489e87bde828af3486a2 2024-12-09T20:43:05,735 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/aee3be9e5710489e87bde828af3486a2, entries=150, sequenceid=919, filesize=12.0 K 2024-12-09T20:43:05,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/4547e5db7f324661856d3bff55d9684e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4547e5db7f324661856d3bff55d9684e 2024-12-09T20:43:05,744 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4547e5db7f324661856d3bff55d9684e, entries=150, sequenceid=919, filesize=12.0 K 2024-12-09T20:43:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/bf9125ac20a54ca6a4ea2e9d0d29398e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/bf9125ac20a54ca6a4ea2e9d0d29398e 2024-12-09T20:43:05,752 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/bf9125ac20a54ca6a4ea2e9d0d29398e, entries=150, sequenceid=919, filesize=12.0 K 2024-12-09T20:43:05,754 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d1100dd05f3cf346719eddc5a46c4983 in 503ms, sequenceid=919, compaction requested=false 2024-12-09T20:43:05,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:05,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:05,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-09T20:43:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-09T20:43:05,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-09T20:43:05,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 659 msec 2024-12-09T20:43:05,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 666 msec 2024-12-09T20:43:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:05,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:43:05,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:05,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:05,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:05,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:05,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:05,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777045805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777045806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777045807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/147b8a0500774ec5be4f879161e69d2c is 50, key is test_row_0/A:col10/1733776985472/Put/seqid=0 2024-12-09T20:43:05,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777045809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777045810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742132_1308 (size=12301) 2024-12-09T20:43:05,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=939 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/147b8a0500774ec5be4f879161e69d2c 2024-12-09T20:43:05,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/1e6780c851d94a64b9ce19b801395f22 is 50, key is test_row_0/B:col10/1733776985472/Put/seqid=0 2024-12-09T20:43:05,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742133_1309 (size=12301) 2024-12-09T20:43:05,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=939 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/1e6780c851d94a64b9ce19b801395f22 2024-12-09T20:43:05,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/83462804f67f445884dc4cc90c83a392 is 50, key is test_row_0/C:col10/1733776985472/Put/seqid=0 2024-12-09T20:43:05,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742134_1310 (size=12301) 2024-12-09T20:43:05,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777045921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777045922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777045922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777045922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777045922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777046130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777046131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777046131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777046131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777046131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T20:43:06,231 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:06,232 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-09T20:43:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T20:43:06,234 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:06,235 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:06,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:06,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=939 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/83462804f67f445884dc4cc90c83a392 2024-12-09T20:43:06,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/147b8a0500774ec5be4f879161e69d2c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/147b8a0500774ec5be4f879161e69d2c 2024-12-09T20:43:06,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/147b8a0500774ec5be4f879161e69d2c, entries=150, sequenceid=939, filesize=12.0 K 2024-12-09T20:43:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/1e6780c851d94a64b9ce19b801395f22 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1e6780c851d94a64b9ce19b801395f22 2024-12-09T20:43:06,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1e6780c851d94a64b9ce19b801395f22, entries=150, sequenceid=939, filesize=12.0 K 2024-12-09T20:43:06,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/83462804f67f445884dc4cc90c83a392 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/83462804f67f445884dc4cc90c83a392 2024-12-09T20:43:06,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/83462804f67f445884dc4cc90c83a392, entries=150, sequenceid=939, filesize=12.0 K 2024-12-09T20:43:06,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d1100dd05f3cf346719eddc5a46c4983 in 528ms, sequenceid=939, compaction requested=true 2024-12-09T20:43:06,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:06,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:06,321 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:06,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:06,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:06,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:06,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:06,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:43:06,322 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:06,323 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:06,323 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/A is initiating minor compaction (all files) 2024-12-09T20:43:06,323 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/A in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:06,323 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ca583fc9d73c46859981baf60fda1d68, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/aee3be9e5710489e87bde828af3486a2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/147b8a0500774ec5be4f879161e69d2c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.8 K 2024-12-09T20:43:06,323 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca583fc9d73c46859981baf60fda1d68, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=902, earliestPutTs=1733776982157 2024-12-09T20:43:06,324 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting aee3be9e5710489e87bde828af3486a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=919, earliestPutTs=1733776983311 2024-12-09T20:43:06,324 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 147b8a0500774ec5be4f879161e69d2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=939, earliestPutTs=1733776985472 2024-12-09T20:43:06,324 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:06,324 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/B is initiating minor compaction (all files) 2024-12-09T20:43:06,324 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/B in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:06,325 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4a928038ecf045d980697dcea335af1c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4547e5db7f324661856d3bff55d9684e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1e6780c851d94a64b9ce19b801395f22] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.8 K 2024-12-09T20:43:06,325 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a928038ecf045d980697dcea335af1c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=902, earliestPutTs=1733776982157 2024-12-09T20:43:06,326 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 4547e5db7f324661856d3bff55d9684e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=919, earliestPutTs=1733776983311 2024-12-09T20:43:06,326 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e6780c851d94a64b9ce19b801395f22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=939, earliestPutTs=1733776985472 2024-12-09T20:43:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T20:43:06,347 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:06,347 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/a912054229504940af4e63b692950f56 is 50, key is test_row_0/A:col10/1733776985472/Put/seqid=0 2024-12-09T20:43:06,355 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#B#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:06,355 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/c7ffcd605d2f437486ea4c347357b7a6 is 50, key is test_row_0/B:col10/1733776985472/Put/seqid=0 2024-12-09T20:43:06,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742135_1311 (size=13187) 2024-12-09T20:43:06,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742136_1312 (size=13187) 2024-12-09T20:43:06,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T20:43:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:06,388 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:43:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:06,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/78f5b0a8c3fd4cf9a3bcbbfb0705fc4a is 50, key is test_row_0/A:col10/1733776985806/Put/seqid=0 2024-12-09T20:43:06,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742137_1313 (size=12301) 2024-12-09T20:43:06,396 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=951 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/78f5b0a8c3fd4cf9a3bcbbfb0705fc4a 2024-12-09T20:43:06,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/5f8c7cca9e0d412d94660664b16df803 is 50, key is test_row_0/B:col10/1733776985806/Put/seqid=0 2024-12-09T20:43:06,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742138_1314 (size=12301) 2024-12-09T20:43:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:06,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777046463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777046464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777046464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777046468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777046468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T20:43:06,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777046571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777046571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777046571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777046581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777046581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,766 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/a912054229504940af4e63b692950f56 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/a912054229504940af4e63b692950f56 2024-12-09T20:43:06,771 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/A of d1100dd05f3cf346719eddc5a46c4983 into a912054229504940af4e63b692950f56(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:06,771 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:06,771 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/A, priority=13, startTime=1733776986321; duration=0sec 2024-12-09T20:43:06,771 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:06,771 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:43:06,771 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:06,773 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:06,773 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/C is initiating minor compaction (all files) 2024-12-09T20:43:06,773 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/C in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:06,773 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/216517747f5543ce9f895ca2eda8af9c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/bf9125ac20a54ca6a4ea2e9d0d29398e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/83462804f67f445884dc4cc90c83a392] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=36.8 K 2024-12-09T20:43:06,773 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216517747f5543ce9f895ca2eda8af9c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=902, earliestPutTs=1733776982157 2024-12-09T20:43:06,774 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9125ac20a54ca6a4ea2e9d0d29398e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=919, earliestPutTs=1733776983311 2024-12-09T20:43:06,774 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/c7ffcd605d2f437486ea4c347357b7a6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/c7ffcd605d2f437486ea4c347357b7a6 2024-12-09T20:43:06,774 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83462804f67f445884dc4cc90c83a392, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=939, earliestPutTs=1733776985472 2024-12-09T20:43:06,779 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/B of d1100dd05f3cf346719eddc5a46c4983 into c7ffcd605d2f437486ea4c347357b7a6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:06,779 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:06,779 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/B, priority=13, startTime=1733776986321; duration=0sec 2024-12-09T20:43:06,779 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:06,779 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:43:06,784 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:06,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777046781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,784 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4d37c1031acf48c9b92b937f7017a945 is 50, key is test_row_0/C:col10/1733776985472/Put/seqid=0 2024-12-09T20:43:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777046781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777046781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742139_1315 (size=13187) 2024-12-09T20:43:06,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777046791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777046791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:06,811 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=951 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/5f8c7cca9e0d412d94660664b16df803 2024-12-09T20:43:06,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/6b968e0639404af9b4df4b206111abdb is 50, key is test_row_0/C:col10/1733776985806/Put/seqid=0 2024-12-09T20:43:06,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742140_1316 (size=12301) 2024-12-09T20:43:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T20:43:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777047091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777047091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777047091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777047100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777047101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,196 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4d37c1031acf48c9b92b937f7017a945 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4d37c1031acf48c9b92b937f7017a945 2024-12-09T20:43:07,202 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/C of d1100dd05f3cf346719eddc5a46c4983 into 4d37c1031acf48c9b92b937f7017a945(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:07,202 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:07,202 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/C, priority=13, startTime=1733776986321; duration=0sec 2024-12-09T20:43:07,202 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:07,202 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:43:07,223 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=951 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/6b968e0639404af9b4df4b206111abdb 2024-12-09T20:43:07,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/78f5b0a8c3fd4cf9a3bcbbfb0705fc4a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/78f5b0a8c3fd4cf9a3bcbbfb0705fc4a 2024-12-09T20:43:07,229 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/78f5b0a8c3fd4cf9a3bcbbfb0705fc4a, entries=150, sequenceid=951, filesize=12.0 K 2024-12-09T20:43:07,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/5f8c7cca9e0d412d94660664b16df803 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/5f8c7cca9e0d412d94660664b16df803 2024-12-09T20:43:07,233 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/5f8c7cca9e0d412d94660664b16df803, entries=150, sequenceid=951, filesize=12.0 K 2024-12-09T20:43:07,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/6b968e0639404af9b4df4b206111abdb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/6b968e0639404af9b4df4b206111abdb 2024-12-09T20:43:07,236 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/6b968e0639404af9b4df4b206111abdb, entries=150, sequenceid=951, filesize=12.0 K 2024-12-09T20:43:07,237 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d1100dd05f3cf346719eddc5a46c4983 in 849ms, sequenceid=951, compaction requested=false 2024-12-09T20:43:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-09T20:43:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-09T20:43:07,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-09T20:43:07,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0030 sec 2024-12-09T20:43:07,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0070 sec 2024-12-09T20:43:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T20:43:07,371 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:07,371 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-09T20:43:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T20:43:07,373 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:07,373 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:07,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T20:43:07,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-09T20:43:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:07,525 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:43:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:07,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/17e003d27746472bafbdf697f9a49966 is 50, key is test_row_0/A:col10/1733776986463/Put/seqid=0 2024-12-09T20:43:07,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742141_1317 (size=12301) 2024-12-09T20:43:07,532 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=974 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/17e003d27746472bafbdf697f9a49966 2024-12-09T20:43:07,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3ab8932541fe47348a768e006455c0b8 is 50, key is test_row_0/B:col10/1733776986463/Put/seqid=0 2024-12-09T20:43:07,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742142_1318 (size=12301) 2024-12-09T20:43:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:07,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:07,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777047611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777047611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777047613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777047614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777047614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T20:43:07,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777047721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777047722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777047722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777047931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777047931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777047931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:07,943 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=974 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3ab8932541fe47348a768e006455c0b8 2024-12-09T20:43:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/af5a19cc9801460a84ca31db3ea10b16 is 50, key is test_row_0/C:col10/1733776986463/Put/seqid=0 2024-12-09T20:43:07,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742143_1319 (size=12301) 2024-12-09T20:43:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T20:43:08,004 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=974 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/af5a19cc9801460a84ca31db3ea10b16 2024-12-09T20:43:08,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/17e003d27746472bafbdf697f9a49966 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17e003d27746472bafbdf697f9a49966 2024-12-09T20:43:08,023 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17e003d27746472bafbdf697f9a49966, entries=150, sequenceid=974, filesize=12.0 K 2024-12-09T20:43:08,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/3ab8932541fe47348a768e006455c0b8 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3ab8932541fe47348a768e006455c0b8 2024-12-09T20:43:08,032 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3ab8932541fe47348a768e006455c0b8, entries=150, sequenceid=974, filesize=12.0 K 2024-12-09T20:43:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/af5a19cc9801460a84ca31db3ea10b16 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/af5a19cc9801460a84ca31db3ea10b16 2024-12-09T20:43:08,040 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/af5a19cc9801460a84ca31db3ea10b16, entries=150, sequenceid=974, filesize=12.0 K 2024-12-09T20:43:08,041 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d1100dd05f3cf346719eddc5a46c4983 in 516ms, sequenceid=974, compaction requested=true 2024-12-09T20:43:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-09T20:43:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-09T20:43:08,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-09T20:43:08,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 670 msec 2024-12-09T20:43:08,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 677 msec 2024-12-09T20:43:08,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:43:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:08,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/53ed2799bcad409899f10415abb619cf is 50, key is test_row_0/A:col10/1733776988253/Put/seqid=0 2024-12-09T20:43:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777048300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777048302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777048303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742144_1320 (size=14741) 2024-12-09T20:43:08,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777048412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777048412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777048413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T20:43:08,511 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:08,512 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-09T20:43:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T20:43:08,515 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:08,516 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:08,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:08,615 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-12-09T20:43:08,615 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:08,615 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,615 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,615 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-12-09T20:43:08,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:08,615 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:08,615 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,615 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:08,618 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-12-09T20:43:08,618 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:08,618 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,618 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:08,619 INFO [Thread-1100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1100. 2024-12-09T20:43:08,619 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-12-09T20:43:08,619 DEBUG [Thread-1100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:08,619 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:08,619 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,619 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,619 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,619 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:08,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:08,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T20:43:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36504 deadline: 1733777048621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777048621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777048622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777048622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,622 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in d1100dd05f3cf346719eddc5a46c4983 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36464 deadline: 1733777048631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,632 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in d1100dd05f3cf346719eddc5a46c4983 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T20:43:08,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:08,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:08,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=987 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/53ed2799bcad409899f10415abb619cf 2024-12-09T20:43:08,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/da7008fdd16442a59171d65b98d4b1ae is 50, key is test_row_0/B:col10/1733776988253/Put/seqid=0 2024-12-09T20:43:08,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742145_1321 (size=12301) 2024-12-09T20:43:08,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T20:43:08,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:08,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:08,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:08,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:08,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T20:43:08,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777048931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777048931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777048931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:08,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T20:43:08,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:08,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:08,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:08,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:08,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T20:43:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:09,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:09,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=987 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/da7008fdd16442a59171d65b98d4b1ae 2024-12-09T20:43:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T20:43:09,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/8869faf5efa14d9cb8d704b11d1623e3 is 50, key is test_row_0/C:col10/1733776988253/Put/seqid=0 2024-12-09T20:43:09,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742146_1322 (size=12301) 2024-12-09T20:43:09,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T20:43:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:09,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T20:43:09,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:09,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. as already flushing 2024-12-09T20:43:09,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:09,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:09,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:09,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:09,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36486 deadline: 1733777049441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36500 deadline: 1733777049441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36484 deadline: 1733777049441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:09,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=987 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/8869faf5efa14d9cb8d704b11d1623e3 2024-12-09T20:43:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/53ed2799bcad409899f10415abb619cf as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/53ed2799bcad409899f10415abb619cf 2024-12-09T20:43:09,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/53ed2799bcad409899f10415abb619cf, entries=200, sequenceid=987, filesize=14.4 K 2024-12-09T20:43:09,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/da7008fdd16442a59171d65b98d4b1ae as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/da7008fdd16442a59171d65b98d4b1ae 2024-12-09T20:43:09,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/da7008fdd16442a59171d65b98d4b1ae, entries=150, sequenceid=987, filesize=12.0 K 2024-12-09T20:43:09,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/8869faf5efa14d9cb8d704b11d1623e3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8869faf5efa14d9cb8d704b11d1623e3 2024-12-09T20:43:09,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8869faf5efa14d9cb8d704b11d1623e3, entries=150, sequenceid=987, filesize=12.0 K 2024-12-09T20:43:09,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d1100dd05f3cf346719eddc5a46c4983 in 1316ms, sequenceid=987, compaction requested=true 2024-12-09T20:43:09,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:09,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:09,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:09,570 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:09,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:09,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:09,570 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:09,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1100dd05f3cf346719eddc5a46c4983:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:09,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:09,571 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:09,571 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:09,571 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/B is initiating minor compaction (all files) 2024-12-09T20:43:09,571 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/A is initiating minor compaction (all files) 2024-12-09T20:43:09,571 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/B in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:09,571 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/A in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:09,572 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/a912054229504940af4e63b692950f56, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/78f5b0a8c3fd4cf9a3bcbbfb0705fc4a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17e003d27746472bafbdf697f9a49966, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/53ed2799bcad409899f10415abb619cf] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=51.3 K 2024-12-09T20:43:09,572 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/c7ffcd605d2f437486ea4c347357b7a6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/5f8c7cca9e0d412d94660664b16df803, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3ab8932541fe47348a768e006455c0b8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/da7008fdd16442a59171d65b98d4b1ae] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=48.9 K 2024-12-09T20:43:09,572 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c7ffcd605d2f437486ea4c347357b7a6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=939, earliestPutTs=1733776985472 2024-12-09T20:43:09,572 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting a912054229504940af4e63b692950f56, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=939, earliestPutTs=1733776985472 2024-12-09T20:43:09,572 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78f5b0a8c3fd4cf9a3bcbbfb0705fc4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=951, earliestPutTs=1733776985806 2024-12-09T20:43:09,572 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8c7cca9e0d412d94660664b16df803, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=951, earliestPutTs=1733776985806 2024-12-09T20:43:09,572 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17e003d27746472bafbdf697f9a49966, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=974, earliestPutTs=1733776986463 2024-12-09T20:43:09,572 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ab8932541fe47348a768e006455c0b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=974, earliestPutTs=1733776986463 2024-12-09T20:43:09,572 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53ed2799bcad409899f10415abb619cf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=987, earliestPutTs=1733776987603 2024-12-09T20:43:09,573 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting da7008fdd16442a59171d65b98d4b1ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=987, earliestPutTs=1733776987607 2024-12-09T20:43:09,586 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:09,587 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/184fe574a2d14140b1ebf4d7ce3ae017 is 50, key is test_row_0/B:col10/1733776988253/Put/seqid=0 2024-12-09T20:43:09,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T20:43:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:09,589 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:43:09,590 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#A#compaction#280 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:09,590 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/c9921d7dee944c62a5d02b07fbf0a287 is 50, key is test_row_0/A:col10/1733776988253/Put/seqid=0 2024-12-09T20:43:09,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/c49697c0b95942cb97352f5fa17f4650 is 50, key is test_row_0/A:col10/1733776988298/Put/seqid=0 2024-12-09T20:43:09,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742148_1324 (size=13323) 2024-12-09T20:43:09,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742147_1323 (size=13323) 2024-12-09T20:43:09,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742149_1325 (size=12301) 2024-12-09T20:43:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T20:43:09,999 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1006 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/c49697c0b95942cb97352f5fa17f4650 2024-12-09T20:43:10,002 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/c9921d7dee944c62a5d02b07fbf0a287 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c9921d7dee944c62a5d02b07fbf0a287 2024-12-09T20:43:10,004 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/184fe574a2d14140b1ebf4d7ce3ae017 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/184fe574a2d14140b1ebf4d7ce3ae017 2024-12-09T20:43:10,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/9bf9c6048547480a830735626246aab8 is 50, key is test_row_0/B:col10/1733776988298/Put/seqid=0 2024-12-09T20:43:10,010 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/A of d1100dd05f3cf346719eddc5a46c4983 into c9921d7dee944c62a5d02b07fbf0a287(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:10,010 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:10,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742150_1326 (size=12301) 2024-12-09T20:43:10,010 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/A, priority=12, startTime=1733776989570; duration=0sec 2024-12-09T20:43:10,010 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:10,010 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:A 2024-12-09T20:43:10,010 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:10,011 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1006 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/9bf9c6048547480a830735626246aab8 2024-12-09T20:43:10,011 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/B of d1100dd05f3cf346719eddc5a46c4983 into 184fe574a2d14140b1ebf4d7ce3ae017(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:10,011 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:10,011 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/B, priority=12, startTime=1733776989570; duration=0sec 2024-12-09T20:43:10,011 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:10,011 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:B 2024-12-09T20:43:10,011 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:10,011 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): d1100dd05f3cf346719eddc5a46c4983/C is initiating minor compaction (all files) 2024-12-09T20:43:10,011 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1100dd05f3cf346719eddc5a46c4983/C in TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:10,011 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4d37c1031acf48c9b92b937f7017a945, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/6b968e0639404af9b4df4b206111abdb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/af5a19cc9801460a84ca31db3ea10b16, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8869faf5efa14d9cb8d704b11d1623e3] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp, totalSize=48.9 K 2024-12-09T20:43:10,012 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d37c1031acf48c9b92b937f7017a945, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=939, earliestPutTs=1733776985472 2024-12-09T20:43:10,012 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b968e0639404af9b4df4b206111abdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=951, earliestPutTs=1733776985806 2024-12-09T20:43:10,012 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting af5a19cc9801460a84ca31db3ea10b16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=974, earliestPutTs=1733776986463 2024-12-09T20:43:10,013 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8869faf5efa14d9cb8d704b11d1623e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=987, earliestPutTs=1733776987607 2024-12-09T20:43:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/58a3e615e79746908fc9a2bff61bc582 is 50, key is test_row_0/C:col10/1733776988298/Put/seqid=0 2024-12-09T20:43:10,019 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1100dd05f3cf346719eddc5a46c4983#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:10,019 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4706ea9441ab4c4db6dbdb48f0dad97d is 50, key is test_row_0/C:col10/1733776988253/Put/seqid=0 2024-12-09T20:43:10,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742151_1327 (size=12301) 2024-12-09T20:43:10,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742152_1328 (size=13323) 2024-12-09T20:43:10,422 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1006 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/58a3e615e79746908fc9a2bff61bc582 2024-12-09T20:43:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/c49697c0b95942cb97352f5fa17f4650 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c49697c0b95942cb97352f5fa17f4650 2024-12-09T20:43:10,435 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/4706ea9441ab4c4db6dbdb48f0dad97d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4706ea9441ab4c4db6dbdb48f0dad97d 2024-12-09T20:43:10,439 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c49697c0b95942cb97352f5fa17f4650, entries=150, sequenceid=1006, filesize=12.0 K 2024-12-09T20:43:10,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/9bf9c6048547480a830735626246aab8 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/9bf9c6048547480a830735626246aab8 2024-12-09T20:43:10,441 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1100dd05f3cf346719eddc5a46c4983/C of d1100dd05f3cf346719eddc5a46c4983 into 4706ea9441ab4c4db6dbdb48f0dad97d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:10,441 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:10,441 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., storeName=d1100dd05f3cf346719eddc5a46c4983/C, priority=12, startTime=1733776989570; duration=0sec 2024-12-09T20:43:10,441 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:10,441 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1100dd05f3cf346719eddc5a46c4983:C 2024-12-09T20:43:10,444 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/9bf9c6048547480a830735626246aab8, entries=150, sequenceid=1006, filesize=12.0 K 2024-12-09T20:43:10,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/58a3e615e79746908fc9a2bff61bc582 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/58a3e615e79746908fc9a2bff61bc582 2024-12-09T20:43:10,448 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/58a3e615e79746908fc9a2bff61bc582, entries=150, sequenceid=1006, filesize=12.0 K 2024-12-09T20:43:10,449 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for d1100dd05f3cf346719eddc5a46c4983 in 860ms, sequenceid=1006, compaction requested=false 2024-12-09T20:43:10,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for d1100dd05f3cf346719eddc5a46c4983: 2024-12-09T20:43:10,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:10,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-09T20:43:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-09T20:43:10,452 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-12-09T20:43:10,452 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:10,452 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,452 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-12-09T20:43:10,452 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:10,452 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-09T20:43:10,452 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:10,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9340 sec 2024-12-09T20:43:10,452 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:10,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.9410 sec 2024-12-09T20:43:10,462 INFO [Thread-1091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1091. 2024-12-09T20:43:10,462 DEBUG [Thread-1091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:10,462 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,463 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:10,642 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-12-09T20:43:10,642 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-12-09T20:43:10,643 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:10,643 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:10,643 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,643 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,643 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,643 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:10,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T20:43:10,662 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 193 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 236 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 184 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 159 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 145 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3399 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3523 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3390 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3414 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3502 2024-12-09T20:43:10,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T20:43:10,662 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:43:10,662 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:43:10,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:10,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:10,664 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T20:43:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T20:43:10,668 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776990667"}]},"ts":"1733776990667"} 2024-12-09T20:43:10,669 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T20:43:10,669 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T20:43:10,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T20:43:10,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1100dd05f3cf346719eddc5a46c4983, UNASSIGN}] 2024-12-09T20:43:10,673 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1100dd05f3cf346719eddc5a46c4983, UNASSIGN 2024-12-09T20:43:10,673 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d1100dd05f3cf346719eddc5a46c4983, regionState=CLOSING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:10,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1100dd05f3cf346719eddc5a46c4983, UNASSIGN because future has completed 2024-12-09T20:43:10,675 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T20:43:10,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:43:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T20:43:10,826 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing d1100dd05f3cf346719eddc5a46c4983, disabling compactions & flushes 2024-12-09T20:43:10,827 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. after waiting 0 ms 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:10,827 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing d1100dd05f3cf346719eddc5a46c4983 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=A 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=B 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:10,827 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1100dd05f3cf346719eddc5a46c4983, store=C 2024-12-09T20:43:10,828 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:10,831 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/c8b8ad9844134a91badfb12b6f222999 is 50, key is test_row_0/A:col10/1733776990461/Put/seqid=0 2024-12-09T20:43:10,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742153_1329 (size=12301) 2024-12-09T20:43:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T20:43:11,236 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1017 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/c8b8ad9844134a91badfb12b6f222999 2024-12-09T20:43:11,242 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/dd14f18140b64c2599b7f996e4789e5a is 50, key is test_row_0/B:col10/1733776990461/Put/seqid=0 2024-12-09T20:43:11,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742154_1330 (size=12301) 2024-12-09T20:43:11,251 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1017 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/dd14f18140b64c2599b7f996e4789e5a 2024-12-09T20:43:11,261 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/7547d0aef9cf4b16baa3c5f84dfea3ad is 50, key is test_row_0/C:col10/1733776990461/Put/seqid=0 2024-12-09T20:43:11,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742155_1331 (size=12301) 2024-12-09T20:43:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T20:43:11,665 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1017 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/7547d0aef9cf4b16baa3c5f84dfea3ad 2024-12-09T20:43:11,669 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/A/c8b8ad9844134a91badfb12b6f222999 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c8b8ad9844134a91badfb12b6f222999 2024-12-09T20:43:11,673 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c8b8ad9844134a91badfb12b6f222999, entries=150, sequenceid=1017, filesize=12.0 K 2024-12-09T20:43:11,674 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/B/dd14f18140b64c2599b7f996e4789e5a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dd14f18140b64c2599b7f996e4789e5a 2024-12-09T20:43:11,686 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dd14f18140b64c2599b7f996e4789e5a, entries=150, sequenceid=1017, filesize=12.0 K 2024-12-09T20:43:11,687 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/.tmp/C/7547d0aef9cf4b16baa3c5f84dfea3ad as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/7547d0aef9cf4b16baa3c5f84dfea3ad 2024-12-09T20:43:11,691 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/7547d0aef9cf4b16baa3c5f84dfea3ad, entries=150, sequenceid=1017, filesize=12.0 K 2024-12-09T20:43:11,692 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d1100dd05f3cf346719eddc5a46c4983 in 865ms, sequenceid=1017, compaction requested=true 2024-12-09T20:43:11,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/177c56a4a2214230a8c1360a1ac329e9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/7018632325ee4c8986d350c7161f9b6e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/507ae0c5499e49d0bde93d65cb5506c3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/4cb1c3095d1a434dbcf190e61ec65e29, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/91c58cc18b174503b6b53b674f6e4cfc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ff98396afbf5422eb2edcab6742131d6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9849d92176d54352871539c84a3fe250, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/661c42a43e064233bdad88970a363b80, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6243f728f7df4c3694bd3988d5e54a91, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/23b789e5d0d54877a68d66fe08ace814, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/1613775d3b5342f2ac9b47e48f204462, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9f172e5e90d545508546bb1f88854c61, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/b14a4cedc870456588f68abe35bff62f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/702e76896e4f49639dc9647214e7dafc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/3eeb52afbbdb47aa9c103787b2f12307, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/22f059a360834ac0b65cc64acc2d1714, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/64d1a2760c114d5e9c28da2aab02918a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/8eb71ad51d144a7cb2419362c1038c47, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17343bf622ab473db76f26a7b5339418, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/d7aa183db3df4e0fa00279e9b2ac027e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/18a84a9dab9041fb9044f391a87e6ed9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/f89c29924c15457c90a97e5aeb1eae47, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6e4103f21bce4d10832cef162939f885, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ca583fc9d73c46859981baf60fda1d68, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/aee3be9e5710489e87bde828af3486a2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/a912054229504940af4e63b692950f56, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/147b8a0500774ec5be4f879161e69d2c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/78f5b0a8c3fd4cf9a3bcbbfb0705fc4a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17e003d27746472bafbdf697f9a49966, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/53ed2799bcad409899f10415abb619cf] to archive 2024-12-09T20:43:11,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:43:11,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/177c56a4a2214230a8c1360a1ac329e9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/177c56a4a2214230a8c1360a1ac329e9 2024-12-09T20:43:11,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/7018632325ee4c8986d350c7161f9b6e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/7018632325ee4c8986d350c7161f9b6e 2024-12-09T20:43:11,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/507ae0c5499e49d0bde93d65cb5506c3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/507ae0c5499e49d0bde93d65cb5506c3 2024-12-09T20:43:11,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/4cb1c3095d1a434dbcf190e61ec65e29 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/4cb1c3095d1a434dbcf190e61ec65e29 2024-12-09T20:43:11,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/91c58cc18b174503b6b53b674f6e4cfc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/91c58cc18b174503b6b53b674f6e4cfc 2024-12-09T20:43:11,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ff98396afbf5422eb2edcab6742131d6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ff98396afbf5422eb2edcab6742131d6 2024-12-09T20:43:11,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9849d92176d54352871539c84a3fe250 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9849d92176d54352871539c84a3fe250 2024-12-09T20:43:11,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/661c42a43e064233bdad88970a363b80 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/661c42a43e064233bdad88970a363b80 2024-12-09T20:43:11,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6243f728f7df4c3694bd3988d5e54a91 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6243f728f7df4c3694bd3988d5e54a91 2024-12-09T20:43:11,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/23b789e5d0d54877a68d66fe08ace814 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/23b789e5d0d54877a68d66fe08ace814 2024-12-09T20:43:11,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/1613775d3b5342f2ac9b47e48f204462 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/1613775d3b5342f2ac9b47e48f204462 2024-12-09T20:43:11,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9f172e5e90d545508546bb1f88854c61 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/9f172e5e90d545508546bb1f88854c61 2024-12-09T20:43:11,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/b14a4cedc870456588f68abe35bff62f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/b14a4cedc870456588f68abe35bff62f 2024-12-09T20:43:11,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/702e76896e4f49639dc9647214e7dafc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/702e76896e4f49639dc9647214e7dafc 2024-12-09T20:43:11,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/3eeb52afbbdb47aa9c103787b2f12307 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/3eeb52afbbdb47aa9c103787b2f12307 2024-12-09T20:43:11,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/22f059a360834ac0b65cc64acc2d1714 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/22f059a360834ac0b65cc64acc2d1714 2024-12-09T20:43:11,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/64d1a2760c114d5e9c28da2aab02918a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/64d1a2760c114d5e9c28da2aab02918a 2024-12-09T20:43:11,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/8eb71ad51d144a7cb2419362c1038c47 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/8eb71ad51d144a7cb2419362c1038c47 2024-12-09T20:43:11,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17343bf622ab473db76f26a7b5339418 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17343bf622ab473db76f26a7b5339418 2024-12-09T20:43:11,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/d7aa183db3df4e0fa00279e9b2ac027e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/d7aa183db3df4e0fa00279e9b2ac027e 2024-12-09T20:43:11,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/18a84a9dab9041fb9044f391a87e6ed9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/18a84a9dab9041fb9044f391a87e6ed9 2024-12-09T20:43:11,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/f89c29924c15457c90a97e5aeb1eae47 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/f89c29924c15457c90a97e5aeb1eae47 2024-12-09T20:43:11,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6e4103f21bce4d10832cef162939f885 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/6e4103f21bce4d10832cef162939f885 2024-12-09T20:43:11,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ca583fc9d73c46859981baf60fda1d68 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/ca583fc9d73c46859981baf60fda1d68 2024-12-09T20:43:11,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/aee3be9e5710489e87bde828af3486a2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/aee3be9e5710489e87bde828af3486a2 2024-12-09T20:43:11,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/a912054229504940af4e63b692950f56 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/a912054229504940af4e63b692950f56 2024-12-09T20:43:11,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/147b8a0500774ec5be4f879161e69d2c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/147b8a0500774ec5be4f879161e69d2c 2024-12-09T20:43:11,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/78f5b0a8c3fd4cf9a3bcbbfb0705fc4a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/78f5b0a8c3fd4cf9a3bcbbfb0705fc4a 2024-12-09T20:43:11,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17e003d27746472bafbdf697f9a49966 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/17e003d27746472bafbdf697f9a49966 2024-12-09T20:43:11,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/53ed2799bcad409899f10415abb619cf to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/53ed2799bcad409899f10415abb619cf 2024-12-09T20:43:11,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/cb0774cb37a14464a459cee08d0b9308, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/146b17c3e6744d60bc98f41394a36acd, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1bf061a90a5d4880acabc1680a04d3af, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6b78f0fc4d9d46448fcf5506fd900c6c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4d15fbf97d904078aeadc41aa27e838d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/d7b2cb4e78dd4bebaa5f16624930f4bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/57ecd69a29ad40788dfee8f21c11704a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3deb7337808e441b8a1d6bbdc118133d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/a7c559ee41194bce9c5743683cc8dccd, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/2e94f3259b43492abe7238b33c07b761, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b0568c967bf54fbabdf7142c5c487d14, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6105c47f81f94c649a453bfb3c52935b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4ca462cb7556418bae74d158d112ce5c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/87524fbff9a64bb88a2deb42313903c3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/77abc15c7e2244a89f22a21958f1bea5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dbf12eda62974e5290a17ada8f46d7a9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/18763f96573f440d9f3cc0d34e4da3d2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/38e377a3676e4f76aec28bdb1a88b5fe, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3d55f58574c44ca683e51520d6faeca2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3afac610941142089d0a8610c355c4df, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/f4bcce0262824f6c859bad0c10c4d1a3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b81d95d4c4d44243822572624081f890, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4a928038ecf045d980697dcea335af1c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/8abbd335420e427390231d5d7ece7010, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4547e5db7f324661856d3bff55d9684e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/c7ffcd605d2f437486ea4c347357b7a6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1e6780c851d94a64b9ce19b801395f22, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/5f8c7cca9e0d412d94660664b16df803, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3ab8932541fe47348a768e006455c0b8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/da7008fdd16442a59171d65b98d4b1ae] to archive 2024-12-09T20:43:11,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:43:11,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/cb0774cb37a14464a459cee08d0b9308 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/cb0774cb37a14464a459cee08d0b9308 2024-12-09T20:43:11,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/146b17c3e6744d60bc98f41394a36acd to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/146b17c3e6744d60bc98f41394a36acd 2024-12-09T20:43:11,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1bf061a90a5d4880acabc1680a04d3af to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1bf061a90a5d4880acabc1680a04d3af 2024-12-09T20:43:11,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6b78f0fc4d9d46448fcf5506fd900c6c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6b78f0fc4d9d46448fcf5506fd900c6c 2024-12-09T20:43:11,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4d15fbf97d904078aeadc41aa27e838d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4d15fbf97d904078aeadc41aa27e838d 2024-12-09T20:43:11,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/d7b2cb4e78dd4bebaa5f16624930f4bb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/d7b2cb4e78dd4bebaa5f16624930f4bb 2024-12-09T20:43:11,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/57ecd69a29ad40788dfee8f21c11704a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/57ecd69a29ad40788dfee8f21c11704a 2024-12-09T20:43:11,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3deb7337808e441b8a1d6bbdc118133d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3deb7337808e441b8a1d6bbdc118133d 2024-12-09T20:43:11,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/a7c559ee41194bce9c5743683cc8dccd to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/a7c559ee41194bce9c5743683cc8dccd 2024-12-09T20:43:11,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/2e94f3259b43492abe7238b33c07b761 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/2e94f3259b43492abe7238b33c07b761 2024-12-09T20:43:11,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b0568c967bf54fbabdf7142c5c487d14 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b0568c967bf54fbabdf7142c5c487d14 2024-12-09T20:43:11,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6105c47f81f94c649a453bfb3c52935b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/6105c47f81f94c649a453bfb3c52935b 2024-12-09T20:43:11,756 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4ca462cb7556418bae74d158d112ce5c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4ca462cb7556418bae74d158d112ce5c 2024-12-09T20:43:11,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/87524fbff9a64bb88a2deb42313903c3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/87524fbff9a64bb88a2deb42313903c3 2024-12-09T20:43:11,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/77abc15c7e2244a89f22a21958f1bea5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/77abc15c7e2244a89f22a21958f1bea5 2024-12-09T20:43:11,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dbf12eda62974e5290a17ada8f46d7a9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dbf12eda62974e5290a17ada8f46d7a9 2024-12-09T20:43:11,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/18763f96573f440d9f3cc0d34e4da3d2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/18763f96573f440d9f3cc0d34e4da3d2 2024-12-09T20:43:11,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/38e377a3676e4f76aec28bdb1a88b5fe to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/38e377a3676e4f76aec28bdb1a88b5fe 2024-12-09T20:43:11,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3d55f58574c44ca683e51520d6faeca2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3d55f58574c44ca683e51520d6faeca2 2024-12-09T20:43:11,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3afac610941142089d0a8610c355c4df to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3afac610941142089d0a8610c355c4df 2024-12-09T20:43:11,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/f4bcce0262824f6c859bad0c10c4d1a3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/f4bcce0262824f6c859bad0c10c4d1a3 2024-12-09T20:43:11,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b81d95d4c4d44243822572624081f890 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/b81d95d4c4d44243822572624081f890 2024-12-09T20:43:11,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4a928038ecf045d980697dcea335af1c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4a928038ecf045d980697dcea335af1c 2024-12-09T20:43:11,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/8abbd335420e427390231d5d7ece7010 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/8abbd335420e427390231d5d7ece7010 2024-12-09T20:43:11,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4547e5db7f324661856d3bff55d9684e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/4547e5db7f324661856d3bff55d9684e 2024-12-09T20:43:11,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/c7ffcd605d2f437486ea4c347357b7a6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/c7ffcd605d2f437486ea4c347357b7a6 2024-12-09T20:43:11,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1e6780c851d94a64b9ce19b801395f22 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/1e6780c851d94a64b9ce19b801395f22 2024-12-09T20:43:11,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/5f8c7cca9e0d412d94660664b16df803 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/5f8c7cca9e0d412d94660664b16df803 2024-12-09T20:43:11,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3ab8932541fe47348a768e006455c0b8 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/3ab8932541fe47348a768e006455c0b8 2024-12-09T20:43:11,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/da7008fdd16442a59171d65b98d4b1ae to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/da7008fdd16442a59171d65b98d4b1ae 2024-12-09T20:43:11,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5a57dcfd4b5a44adb7b0e03e535cc630, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/75dd46cd05c74fb78d15c8ac35d63302, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/84b32028087249c89232f0335c888b7a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c3a5fdd3b5f84a8a9789a6989a86798f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/223f2a04c4564816abd5a4c2964a9938, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/e580ffc8736f413586817ca934a3aa10, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/58349ea7aed9475cbfc6f29445cda437, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/ae03f45742ff4ccf87cba73d11406e7a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2f744a0587644584a50deb11f0301b99, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/38869224258f40a48dd347a640de7128, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/905254bbbd3b46ce95ad71fb6bf9df00, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5ebf0a242ea542b8bb4df05048c653d4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1f50de20262249a89ba3fd2f4467fff0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c5ba6c8301cc4a8291b43ee16d416d94, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4cdcf0c77b094a0ba3dcdbc8fa07f137, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4ee8c3840c084f3fa73e4da9d7ab26b3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2c34375c89f94d36b1090b51fd0b551f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/39c344a15647489eabab6c7221f801dd, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1321884cce7944128be69f75319c6637, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/03dd75a61502430f9ae7afff1f122944, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8f82f06f522b47228873b0aade6c0b7d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/dae4599408874a60ab3ded936c00dd43, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/216517747f5543ce9f895ca2eda8af9c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/0a51969cb4794e379c72c1f9f78c9085, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/bf9125ac20a54ca6a4ea2e9d0d29398e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4d37c1031acf48c9b92b937f7017a945, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/83462804f67f445884dc4cc90c83a392, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/6b968e0639404af9b4df4b206111abdb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/af5a19cc9801460a84ca31db3ea10b16, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8869faf5efa14d9cb8d704b11d1623e3] to archive 2024-12-09T20:43:11,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:43:11,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5a57dcfd4b5a44adb7b0e03e535cc630 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5a57dcfd4b5a44adb7b0e03e535cc630 2024-12-09T20:43:11,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/75dd46cd05c74fb78d15c8ac35d63302 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/75dd46cd05c74fb78d15c8ac35d63302 2024-12-09T20:43:11,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/84b32028087249c89232f0335c888b7a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/84b32028087249c89232f0335c888b7a 2024-12-09T20:43:11,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c3a5fdd3b5f84a8a9789a6989a86798f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c3a5fdd3b5f84a8a9789a6989a86798f 2024-12-09T20:43:11,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/223f2a04c4564816abd5a4c2964a9938 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/223f2a04c4564816abd5a4c2964a9938 2024-12-09T20:43:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T20:43:11,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/e580ffc8736f413586817ca934a3aa10 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/e580ffc8736f413586817ca934a3aa10 2024-12-09T20:43:11,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/58349ea7aed9475cbfc6f29445cda437 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/58349ea7aed9475cbfc6f29445cda437 2024-12-09T20:43:11,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/ae03f45742ff4ccf87cba73d11406e7a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/ae03f45742ff4ccf87cba73d11406e7a 2024-12-09T20:43:11,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2f744a0587644584a50deb11f0301b99 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2f744a0587644584a50deb11f0301b99 2024-12-09T20:43:11,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/38869224258f40a48dd347a640de7128 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/38869224258f40a48dd347a640de7128 2024-12-09T20:43:11,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/905254bbbd3b46ce95ad71fb6bf9df00 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/905254bbbd3b46ce95ad71fb6bf9df00 2024-12-09T20:43:11,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5ebf0a242ea542b8bb4df05048c653d4 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/5ebf0a242ea542b8bb4df05048c653d4 2024-12-09T20:43:11,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1f50de20262249a89ba3fd2f4467fff0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1f50de20262249a89ba3fd2f4467fff0 2024-12-09T20:43:11,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c5ba6c8301cc4a8291b43ee16d416d94 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/c5ba6c8301cc4a8291b43ee16d416d94 2024-12-09T20:43:11,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4cdcf0c77b094a0ba3dcdbc8fa07f137 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4cdcf0c77b094a0ba3dcdbc8fa07f137 2024-12-09T20:43:11,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4ee8c3840c084f3fa73e4da9d7ab26b3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4ee8c3840c084f3fa73e4da9d7ab26b3 2024-12-09T20:43:11,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2c34375c89f94d36b1090b51fd0b551f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/2c34375c89f94d36b1090b51fd0b551f 2024-12-09T20:43:11,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/39c344a15647489eabab6c7221f801dd to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/39c344a15647489eabab6c7221f801dd 2024-12-09T20:43:11,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1321884cce7944128be69f75319c6637 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/1321884cce7944128be69f75319c6637 2024-12-09T20:43:11,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/03dd75a61502430f9ae7afff1f122944 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/03dd75a61502430f9ae7afff1f122944 2024-12-09T20:43:11,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8f82f06f522b47228873b0aade6c0b7d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8f82f06f522b47228873b0aade6c0b7d 2024-12-09T20:43:11,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/dae4599408874a60ab3ded936c00dd43 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/dae4599408874a60ab3ded936c00dd43 2024-12-09T20:43:11,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/216517747f5543ce9f895ca2eda8af9c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/216517747f5543ce9f895ca2eda8af9c 2024-12-09T20:43:11,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/0a51969cb4794e379c72c1f9f78c9085 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/0a51969cb4794e379c72c1f9f78c9085 2024-12-09T20:43:11,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/bf9125ac20a54ca6a4ea2e9d0d29398e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/bf9125ac20a54ca6a4ea2e9d0d29398e 2024-12-09T20:43:11,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4d37c1031acf48c9b92b937f7017a945 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4d37c1031acf48c9b92b937f7017a945 2024-12-09T20:43:11,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/83462804f67f445884dc4cc90c83a392 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/83462804f67f445884dc4cc90c83a392 2024-12-09T20:43:11,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/6b968e0639404af9b4df4b206111abdb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/6b968e0639404af9b4df4b206111abdb 2024-12-09T20:43:11,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/af5a19cc9801460a84ca31db3ea10b16 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/af5a19cc9801460a84ca31db3ea10b16 2024-12-09T20:43:11,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8869faf5efa14d9cb8d704b11d1623e3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/8869faf5efa14d9cb8d704b11d1623e3 2024-12-09T20:43:11,850 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/recovered.edits/1020.seqid, newMaxSeqId=1020, maxSeqId=1 2024-12-09T20:43:11,851 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. 2024-12-09T20:43:11,851 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for d1100dd05f3cf346719eddc5a46c4983: Waiting for close lock at 1733776990827Running coprocessor pre-close hooks at 1733776990827Disabling compacts and flushes for region at 1733776990827Disabling writes for close at 1733776990827Obtaining lock to block concurrent updates at 1733776990827Preparing flush snapshotting stores in d1100dd05f3cf346719eddc5a46c4983 at 1733776990827Finished memstore snapshotting TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733776990828 (+1 ms)Flushing stores of TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983. at 1733776990828Flushing d1100dd05f3cf346719eddc5a46c4983/A: creating writer at 1733776990828Flushing d1100dd05f3cf346719eddc5a46c4983/A: appending metadata at 1733776990831 (+3 ms)Flushing d1100dd05f3cf346719eddc5a46c4983/A: closing flushed file at 1733776990831Flushing d1100dd05f3cf346719eddc5a46c4983/B: creating writer at 1733776991239 (+408 ms)Flushing d1100dd05f3cf346719eddc5a46c4983/B: appending metadata at 1733776991242 (+3 ms)Flushing d1100dd05f3cf346719eddc5a46c4983/B: closing flushed file at 1733776991242Flushing d1100dd05f3cf346719eddc5a46c4983/C: creating writer at 1733776991256 (+14 ms)Flushing d1100dd05f3cf346719eddc5a46c4983/C: appending metadata at 1733776991260 (+4 ms)Flushing d1100dd05f3cf346719eddc5a46c4983/C: closing flushed file at 1733776991261 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74dc34d5: reopening flushed file at 1733776991669 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f04f0f7: reopening flushed file at 1733776991674 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ccc94b2: reopening flushed file at 1733776991687 (+13 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d1100dd05f3cf346719eddc5a46c4983 in 865ms, sequenceid=1017, compaction requested=true at 1733776991692 (+5 ms)Writing region close event to WAL at 1733776991847 (+155 ms)Running coprocessor post-close hooks at 1733776991851 (+4 ms)Closed at 1733776991851 2024-12-09T20:43:11,853 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:11,854 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d1100dd05f3cf346719eddc5a46c4983, regionState=CLOSED 2024-12-09T20:43:11,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:43:11,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-09T20:43:11,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure d1100dd05f3cf346719eddc5a46c4983, server=00c23a7251f8,36519,1733776895755 in 1.1810 sec 2024-12-09T20:43:11,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-09T20:43:11,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1100dd05f3cf346719eddc5a46c4983, UNASSIGN in 1.1860 sec 2024-12-09T20:43:11,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-09T20:43:11,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1900 sec 2024-12-09T20:43:11,863 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776991863"}]},"ts":"1733776991863"} 2024-12-09T20:43:11,864 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T20:43:11,864 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T20:43:11,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2010 sec 2024-12-09T20:43:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T20:43:12,821 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:12,822 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T20:43:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:12,824 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-09T20:43:12,824 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:12,836 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:12,839 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/recovered.edits] 2024-12-09T20:43:12,844 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c49697c0b95942cb97352f5fa17f4650 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c49697c0b95942cb97352f5fa17f4650 2024-12-09T20:43:12,846 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c8b8ad9844134a91badfb12b6f222999 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c8b8ad9844134a91badfb12b6f222999 2024-12-09T20:43:12,847 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c9921d7dee944c62a5d02b07fbf0a287 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/A/c9921d7dee944c62a5d02b07fbf0a287 2024-12-09T20:43:12,850 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/184fe574a2d14140b1ebf4d7ce3ae017 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/184fe574a2d14140b1ebf4d7ce3ae017 2024-12-09T20:43:12,852 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/9bf9c6048547480a830735626246aab8 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/9bf9c6048547480a830735626246aab8 2024-12-09T20:43:12,854 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dd14f18140b64c2599b7f996e4789e5a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/B/dd14f18140b64c2599b7f996e4789e5a 2024-12-09T20:43:12,857 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4706ea9441ab4c4db6dbdb48f0dad97d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/4706ea9441ab4c4db6dbdb48f0dad97d 2024-12-09T20:43:12,858 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/58a3e615e79746908fc9a2bff61bc582 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/58a3e615e79746908fc9a2bff61bc582 2024-12-09T20:43:12,860 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/7547d0aef9cf4b16baa3c5f84dfea3ad to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/C/7547d0aef9cf4b16baa3c5f84dfea3ad 2024-12-09T20:43:12,863 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/recovered.edits/1020.seqid to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983/recovered.edits/1020.seqid 2024-12-09T20:43:12,864 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/d1100dd05f3cf346719eddc5a46c4983 2024-12-09T20:43:12,864 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T20:43:12,866 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:12,869 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T20:43:12,871 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T20:43:12,872 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:12,873 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T20:43:12,873 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733776992873"}]},"ts":"9223372036854775807"} 2024-12-09T20:43:12,875 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T20:43:12,875 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d1100dd05f3cf346719eddc5a46c4983, NAME => 'TestAcidGuarantees,,1733776966385.d1100dd05f3cf346719eddc5a46c4983.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T20:43:12,875 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T20:43:12,875 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733776992875"}]},"ts":"9223372036854775807"} 2024-12-09T20:43:12,878 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T20:43:12,879 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:12,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-12-09T20:43:12,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-09T20:43:12,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T20:43:12,934 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:12,943 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_421848342_22 at /127.0.0.1:37728 [Waiting for operation #276] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_919247220_22 at /127.0.0.1:57406 [Waiting for operation #333] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=587 (was 455) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2893 (was 3739) 2024-12-09T20:43:12,954 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=587, ProcessCount=11, AvailableMemoryMB=2892 2024-12-09T20:43:12,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T20:43:12,958 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:43:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:12,961 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:43:12,961 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:12,965 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-12-09T20:43:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-09T20:43:12,976 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:43:12,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742156_1332 (size=1061) 2024-12-09T20:43:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-09T20:43:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-09T20:43:13,385 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948 2024-12-09T20:43:13,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742157_1333 (size=53) 2024-12-09T20:43:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-09T20:43:13,791 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:43:13,791 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f4469ee439a16ac91972d18bc523679b, disabling compactions & flushes 2024-12-09T20:43:13,791 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:13,791 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:13,791 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. after waiting 0 ms 2024-12-09T20:43:13,791 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:13,791 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:13,792 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f4469ee439a16ac91972d18bc523679b: Waiting for close lock at 1733776993791Disabling compacts and flushes for region at 1733776993791Disabling writes for close at 1733776993791Writing region close event to WAL at 1733776993791Closed at 1733776993791 2024-12-09T20:43:13,793 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:43:13,793 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733776993793"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776993793"}]},"ts":"1733776993793"} 2024-12-09T20:43:13,795 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:43:13,796 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:43:13,796 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776993796"}]},"ts":"1733776993796"} 2024-12-09T20:43:13,798 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T20:43:13,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, ASSIGN}] 2024-12-09T20:43:13,799 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, ASSIGN 2024-12-09T20:43:13,800 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, ASSIGN; state=OFFLINE, location=00c23a7251f8,36519,1733776895755; forceNewPlan=false, retain=false 2024-12-09T20:43:13,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=f4469ee439a16ac91972d18bc523679b, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:13,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, ASSIGN because future has completed 2024-12-09T20:43:13,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:43:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-09T20:43:14,105 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:14,105 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:43:14,106 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,106 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:43:14,106 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,106 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,107 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,108 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:43:14,108 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4469ee439a16ac91972d18bc523679b columnFamilyName A 2024-12-09T20:43:14,108 DEBUG [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:14,109 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(327): Store=f4469ee439a16ac91972d18bc523679b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:43:14,109 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,110 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:43:14,110 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4469ee439a16ac91972d18bc523679b columnFamilyName B 2024-12-09T20:43:14,110 DEBUG [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:14,110 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(327): Store=f4469ee439a16ac91972d18bc523679b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:43:14,110 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,111 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:43:14,111 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4469ee439a16ac91972d18bc523679b columnFamilyName C 2024-12-09T20:43:14,111 DEBUG [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:14,112 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(327): Store=f4469ee439a16ac91972d18bc523679b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:43:14,112 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:14,112 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,112 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,112 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,113 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,113 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,114 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:43:14,115 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,117 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:43:14,117 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened f4469ee439a16ac91972d18bc523679b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73562786, jitterRate=0.09617093205451965}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:43:14,117 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:14,118 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for f4469ee439a16ac91972d18bc523679b: Running coprocessor pre-open hook at 1733776994106Writing region info on filesystem at 1733776994106Initializing all the Stores at 1733776994107 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776994107Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776994107Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776994107Cleaning up temporary data from old regions at 1733776994113 (+6 ms)Running coprocessor post-open hooks at 1733776994117 (+4 ms)Region opened successfully at 1733776994118 (+1 ms) 2024-12-09T20:43:14,119 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., pid=91, masterSystemTime=1733776994103 2024-12-09T20:43:14,121 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:14,121 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:14,121 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=f4469ee439a16ac91972d18bc523679b, regionState=OPEN, openSeqNum=2, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:14,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:43:14,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-09T20:43:14,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 in 171 msec 2024-12-09T20:43:14,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-09T20:43:14,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, ASSIGN in 328 msec 2024-12-09T20:43:14,128 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:43:14,128 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776994128"}]},"ts":"1733776994128"} 2024-12-09T20:43:14,130 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T20:43:14,131 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:43:14,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-12-09T20:43:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-09T20:43:15,122 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:15,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:15,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:15,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:15,124 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:15,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:15,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366e40df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:15,126 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65307808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:15,127 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:15,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:15,130 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T20:43:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c23a7251f8,42351,1733776895022 2024-12-09T20:43:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21f9e0dd 2024-12-09T20:43:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T20:43:15,132 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T20:43:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T20:43:15,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:43:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:15,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742158_1334 (size=1097) 2024-12-09T20:43:15,552 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T20:43:15,552 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T20:43:15,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T20:43:15,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, REOPEN/MOVE}] 2024-12-09T20:43:15,559 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, REOPEN/MOVE 2024-12-09T20:43:15,560 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=f4469ee439a16ac91972d18bc523679b, regionState=CLOSING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:15,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, REOPEN/MOVE because future has completed 2024-12-09T20:43:15,562 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T20:43:15,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:43:15,715 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:15,715 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T20:43:15,715 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing f4469ee439a16ac91972d18bc523679b, disabling compactions & flushes 2024-12-09T20:43:15,715 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:15,715 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:15,715 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. after waiting 0 ms 2024-12-09T20:43:15,715 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:15,777 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T20:43:15,778 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:15,778 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for f4469ee439a16ac91972d18bc523679b: Waiting for close lock at 1733776995715Running coprocessor pre-close hooks at 1733776995715Disabling compacts and flushes for region at 1733776995715Disabling writes for close at 1733776995715Writing region close event to WAL at 1733776995716 (+1 ms)Running coprocessor post-close hooks at 1733776995778 (+62 ms)Closed at 1733776995778 2024-12-09T20:43:15,778 WARN [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: f4469ee439a16ac91972d18bc523679b to self. 2024-12-09T20:43:15,780 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:15,781 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=f4469ee439a16ac91972d18bc523679b, regionState=CLOSED 2024-12-09T20:43:15,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:43:15,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-09T20:43:15,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 in 222 msec 2024-12-09T20:43:15,786 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, REOPEN/MOVE; state=CLOSED, location=00c23a7251f8,36519,1733776895755; forceNewPlan=false, retain=true 2024-12-09T20:43:15,937 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=f4469ee439a16ac91972d18bc523679b, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:15,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, REOPEN/MOVE because future has completed 2024-12-09T20:43:15,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:43:16,095 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,095 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:43:16,095 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,095 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:43:16,095 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,095 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,097 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,098 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:43:16,098 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4469ee439a16ac91972d18bc523679b columnFamilyName A 2024-12-09T20:43:16,100 DEBUG [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:16,100 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(327): Store=f4469ee439a16ac91972d18bc523679b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:43:16,101 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,102 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:43:16,102 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4469ee439a16ac91972d18bc523679b columnFamilyName B 2024-12-09T20:43:16,102 DEBUG [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:16,102 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(327): Store=f4469ee439a16ac91972d18bc523679b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:43:16,102 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,103 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:43:16,103 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4469ee439a16ac91972d18bc523679b columnFamilyName C 2024-12-09T20:43:16,103 DEBUG [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:16,104 INFO [StoreOpener-f4469ee439a16ac91972d18bc523679b-1 {}] regionserver.HStore(327): Store=f4469ee439a16ac91972d18bc523679b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:43:16,104 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,104 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,105 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,106 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,107 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,107 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,107 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:43:16,109 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,110 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened f4469ee439a16ac91972d18bc523679b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75104235, jitterRate=0.11914031207561493}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:43:16,110 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,111 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for f4469ee439a16ac91972d18bc523679b: Running coprocessor pre-open hook at 1733776996096Writing region info on filesystem at 1733776996096Initializing all the Stores at 1733776996096Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776996096Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776996097 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776996097Cleaning up temporary data from old regions at 1733776996107 (+10 ms)Running coprocessor post-open hooks at 1733776996110 (+3 ms)Region opened successfully at 1733776996111 (+1 ms) 2024-12-09T20:43:16,112 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., pid=96, masterSystemTime=1733776996091 2024-12-09T20:43:16,114 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,114 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,114 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=f4469ee439a16ac91972d18bc523679b, regionState=OPEN, openSeqNum=5, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:43:16,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-12-09T20:43:16,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 in 179 msec 2024-12-09T20:43:16,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-09T20:43:16,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, REOPEN/MOVE in 562 msec 2024-12-09T20:43:16,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-09T20:43:16,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 569 msec 2024-12-09T20:43:16,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 995 msec 2024-12-09T20:43:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-09T20:43:16,133 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:16,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,142 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a964850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,144 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db3d7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,147 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@637096e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f489787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,152 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba8dd8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,154 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58826222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,158 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ef3341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,160 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4892a380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,163 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1ea44a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,165 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263d4847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,167 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f6fd67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,169 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dcc831e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,174 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140977dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,176 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd49e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,178 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d2b3d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,180 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e5f79d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,186 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192d35db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,188 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c8af8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:16,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:16,191 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717ec0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:16,193 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a907c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:16,194 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,205 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,217 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,217 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-09T20:43:16,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T20:43:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,223 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:16,224 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:16,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:16,237 DEBUG [Thread-1529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,241 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,253 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:43:16,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:16,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:16,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:16,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:16,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:16,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:16,273 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,289 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,291 DEBUG [Thread-1525 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,291 DEBUG [Thread-1525 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,305 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,316 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,329 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:16,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776996304, max timestamp=1733776996326Num uniques 859; 2024-12-09T20:43:16,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776996304, max timestamp=1733776996326Num uniques 859; 2024-12-09T20:43:16,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:16,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T20:43:16,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733776996304, max timestamp=1733776996326Num uniques 859; 2024-12-09T20:43:16,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:16,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:43:16,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120955330a9e74e14aee829cad4d5be66936_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733776996265/Put/seqid=0 2024-12-09T20:43:16,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742159_1335 (size=12154) 2024-12-09T20:43:16,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:16,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T20:43:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120955330a9e74e14aee829cad4d5be66936_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120955330a9e74e14aee829cad4d5be66936_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,379 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/c55432042af94d00922ec36d33b56fd8, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:16,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/c55432042af94d00922ec36d33b56fd8 is 175, key is test_row_0/A:col10/1733776996265/Put/seqid=0 2024-12-09T20:43:16,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742160_1336 (size=30955) 2024-12-09T20:43:16,389 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/c55432042af94d00922ec36d33b56fd8 2024-12-09T20:43:16,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/a57eeee09d604d0a8163cb2fca223efa is 50, key is test_row_0/B:col10/1733776996265/Put/seqid=0 2024-12-09T20:43:16,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742161_1337 (size=12001) 2024-12-09T20:43:16,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/a57eeee09d604d0a8163cb2fca223efa 2024-12-09T20:43:16,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733776996407, max timestamp=1733776996428Num uniques 1018; 2024-12-09T20:43:16,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:43:16,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733776996407, max timestamp=1733776996428Num uniques 1018; 2024-12-09T20:43:16,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:43:16,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733776996407, max timestamp=1733776996428Num uniques 1018; 2024-12-09T20:43:16,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:43:16,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/6a99e18bf4734257bfc6028097f1606f is 50, key is test_row_0/C:col10/1733776996265/Put/seqid=0 2024-12-09T20:43:16,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742162_1338 (size=12001) 2024-12-09T20:43:16,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/6a99e18bf4734257bfc6028097f1606f 2024-12-09T20:43:16,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/c55432042af94d00922ec36d33b56fd8 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/c55432042af94d00922ec36d33b56fd8 2024-12-09T20:43:16,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/c55432042af94d00922ec36d33b56fd8, entries=150, sequenceid=14, filesize=30.2 K 2024-12-09T20:43:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/a57eeee09d604d0a8163cb2fca223efa as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/a57eeee09d604d0a8163cb2fca223efa 2024-12-09T20:43:16,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/a57eeee09d604d0a8163cb2fca223efa, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T20:43:16,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/6a99e18bf4734257bfc6028097f1606f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6a99e18bf4734257bfc6028097f1606f 2024-12-09T20:43:16,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6a99e18bf4734257bfc6028097f1606f, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T20:43:16,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.63 KB/103050 for f4469ee439a16ac91972d18bc523679b in 256ms, sequenceid=14, compaction requested=false 2024-12-09T20:43:16,523 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T20:43:16,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=114.05 KB heapSize=324.38 KB 2024-12-09T20:43:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:16,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T20:43:16,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:16,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T20:43:16,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209299db223fb134458a4fd0037b459b9c8_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733776996480/Put/seqid=0 2024-12-09T20:43:16,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777056564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742163_1339 (size=17034) 2024-12-09T20:43:16,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777056564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777056567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777056569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777056569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T20:43:16,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:16,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777056681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777056691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777056691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777056691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777056691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T20:43:16,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:16,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T20:43:16,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777056901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777056912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777056912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777056913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777056913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:16,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:16,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209299db223fb134458a4fd0037b459b9c8_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209299db223fb134458a4fd0037b459b9c8_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:16,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e26a01ecd0a741d18fc8c3a827ff6d2b, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:16,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e26a01ecd0a741d18fc8c3a827ff6d2b is 175, key is test_row_0/A:col10/1733776996480/Put/seqid=0 2024-12-09T20:43:16,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T20:43:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:16,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742164_1340 (size=48139) 2024-12-09T20:43:17,011 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e26a01ecd0a741d18fc8c3a827ff6d2b 2024-12-09T20:43:17,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/f566dd219c4846b79fb822685af6ba05 is 50, key is test_row_0/B:col10/1733776996480/Put/seqid=0 2024-12-09T20:43:17,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742165_1341 (size=12001) 2024-12-09T20:43:17,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T20:43:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:17,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777057221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777057231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777057231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777057231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777057231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T20:43:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:17,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T20:43:17,376 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:43:17,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/f566dd219c4846b79fb822685af6ba05 2024-12-09T20:43:17,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/4c0d624b6e1e48528dc6eb32f000c0f6 is 50, key is test_row_0/C:col10/1733776996480/Put/seqid=0 2024-12-09T20:43:17,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T20:43:17,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:17,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:17,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:17,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:17,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742166_1342 (size=12001) 2024-12-09T20:43:17,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/4c0d624b6e1e48528dc6eb32f000c0f6 2024-12-09T20:43:17,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e26a01ecd0a741d18fc8c3a827ff6d2b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e26a01ecd0a741d18fc8c3a827ff6d2b 2024-12-09T20:43:17,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e26a01ecd0a741d18fc8c3a827ff6d2b, entries=250, sequenceid=65, filesize=47.0 K 2024-12-09T20:43:17,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/f566dd219c4846b79fb822685af6ba05 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/f566dd219c4846b79fb822685af6ba05 2024-12-09T20:43:17,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/f566dd219c4846b79fb822685af6ba05, entries=150, sequenceid=65, filesize=11.7 K 2024-12-09T20:43:17,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/4c0d624b6e1e48528dc6eb32f000c0f6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4c0d624b6e1e48528dc6eb32f000c0f6 2024-12-09T20:43:17,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4c0d624b6e1e48528dc6eb32f000c0f6, entries=150, sequenceid=65, filesize=11.7 K 2024-12-09T20:43:17,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~324.33 KB/332112, currentSize=73.80 KB/75570 for f4469ee439a16ac91972d18bc523679b in 980ms, sequenceid=65, compaction requested=false 2024-12-09T20:43:17,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:17,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T20:43:17,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:17,609 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T20:43:17,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:17,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:17,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:17,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:17,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:17,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:17,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120960dfc2dfa75c45838da2c49b4684ab8b_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733776996567/Put/seqid=0 2024-12-09T20:43:17,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742167_1343 (size=12154) 2024-12-09T20:43:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:17,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777057789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777057789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777057791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777057804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777057804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777057911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777057911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777057911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777057912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777057912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:18,084 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120960dfc2dfa75c45838da2c49b4684ab8b_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120960dfc2dfa75c45838da2c49b4684ab8b_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:18,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/f93a70f707a849558b51be9a3e5ea4cf, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:18,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/f93a70f707a849558b51be9a3e5ea4cf is 175, key is test_row_0/A:col10/1733776996567/Put/seqid=0 2024-12-09T20:43:18,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742168_1344 (size=30955) 2024-12-09T20:43:18,109 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/f93a70f707a849558b51be9a3e5ea4cf 2024-12-09T20:43:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/99fb21457d75434d9a310c77bcc85a16 is 50, key is test_row_0/B:col10/1733776996567/Put/seqid=0 2024-12-09T20:43:18,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777058122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777058122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777058122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777058122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777058122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742169_1345 (size=12001) 2024-12-09T20:43:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T20:43:18,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777058442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777058442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777058443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777058443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777058443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,544 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/99fb21457d75434d9a310c77bcc85a16 2024-12-09T20:43:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/57f01127f71e40349cf3dcaa97a52cf9 is 50, key is test_row_0/C:col10/1733776996567/Put/seqid=0 2024-12-09T20:43:18,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742170_1346 (size=12001) 2024-12-09T20:43:18,567 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/57f01127f71e40349cf3dcaa97a52cf9 2024-12-09T20:43:18,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/f93a70f707a849558b51be9a3e5ea4cf as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f93a70f707a849558b51be9a3e5ea4cf 2024-12-09T20:43:18,576 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f93a70f707a849558b51be9a3e5ea4cf, entries=150, sequenceid=79, filesize=30.2 K 2024-12-09T20:43:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/99fb21457d75434d9a310c77bcc85a16 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/99fb21457d75434d9a310c77bcc85a16 2024-12-09T20:43:18,582 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/99fb21457d75434d9a310c77bcc85a16, entries=150, sequenceid=79, filesize=11.7 K 2024-12-09T20:43:18,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/57f01127f71e40349cf3dcaa97a52cf9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/57f01127f71e40349cf3dcaa97a52cf9 2024-12-09T20:43:18,589 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/57f01127f71e40349cf3dcaa97a52cf9, entries=150, sequenceid=79, filesize=11.7 K 2024-12-09T20:43:18,590 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f4469ee439a16ac91972d18bc523679b in 981ms, sequenceid=79, compaction requested=true 2024-12-09T20:43:18,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:18,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:18,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-09T20:43:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-09T20:43:18,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-09T20:43:18,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3680 sec 2024-12-09T20:43:18,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.3770 sec 2024-12-09T20:43:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:18,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:43:18,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:18,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:18,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:18,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:18,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:18,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:18,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097518805bd8474d6f80ab9f9c1d715f61_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733776997802/Put/seqid=0 2024-12-09T20:43:18,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742171_1347 (size=14594) 2024-12-09T20:43:18,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777058969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777058973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777058973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777058977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777058979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777059081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777059081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777059091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777059092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777059101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777059291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777059291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777059310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777059311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777059311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,368 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:19,370 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097518805bd8474d6f80ab9f9c1d715f61_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097518805bd8474d6f80ab9f9c1d715f61_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:19,371 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/2de4d5a6b86a401fbf398f1cebde8db8, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:19,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/2de4d5a6b86a401fbf398f1cebde8db8 is 175, key is test_row_0/A:col10/1733776997802/Put/seqid=0 2024-12-09T20:43:19,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742172_1348 (size=39549) 2024-12-09T20:43:19,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777059601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777059601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777059621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777059621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777059622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:19,775 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/2de4d5a6b86a401fbf398f1cebde8db8 2024-12-09T20:43:19,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/8d4f0930d38e4ca2ac9c555cf7b543d2 is 50, key is test_row_0/B:col10/1733776997802/Put/seqid=0 2024-12-09T20:43:19,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742173_1349 (size=12001) 2024-12-09T20:43:20,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777060121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:20,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777060121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:20,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777060131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:20,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777060131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:20,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777060141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:20,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/8d4f0930d38e4ca2ac9c555cf7b543d2 2024-12-09T20:43:20,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/3ca4f2d1867e400eaaa23b5e801b55dd is 50, key is test_row_0/C:col10/1733776997802/Put/seqid=0 2024-12-09T20:43:20,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742174_1350 (size=12001) 2024-12-09T20:43:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T20:43:20,381 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:20,381 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-09T20:43:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T20:43:20,383 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:20,383 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:20,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T20:43:20,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T20:43:20,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:20,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:20,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/3ca4f2d1867e400eaaa23b5e801b55dd 2024-12-09T20:43:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/2de4d5a6b86a401fbf398f1cebde8db8 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/2de4d5a6b86a401fbf398f1cebde8db8 2024-12-09T20:43:20,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/2de4d5a6b86a401fbf398f1cebde8db8, entries=200, sequenceid=99, filesize=38.6 K 2024-12-09T20:43:20,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/8d4f0930d38e4ca2ac9c555cf7b543d2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8d4f0930d38e4ca2ac9c555cf7b543d2 2024-12-09T20:43:20,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8d4f0930d38e4ca2ac9c555cf7b543d2, entries=150, sequenceid=99, filesize=11.7 K 2024-12-09T20:43:20,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/3ca4f2d1867e400eaaa23b5e801b55dd as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/3ca4f2d1867e400eaaa23b5e801b55dd 2024-12-09T20:43:20,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/3ca4f2d1867e400eaaa23b5e801b55dd, entries=150, sequenceid=99, filesize=11.7 K 2024-12-09T20:43:20,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f4469ee439a16ac91972d18bc523679b in 1660ms, sequenceid=99, compaction requested=true 2024-12-09T20:43:20,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:20,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:20,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:20,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:20,616 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:20,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:20,616 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:20,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:20,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:20,617 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:20,617 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/B is initiating minor compaction (all files) 2024-12-09T20:43:20,617 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/B in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:20,617 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/a57eeee09d604d0a8163cb2fca223efa, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/f566dd219c4846b79fb822685af6ba05, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/99fb21457d75434d9a310c77bcc85a16, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8d4f0930d38e4ca2ac9c555cf7b543d2] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=46.9 K 2024-12-09T20:43:20,617 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting a57eeee09d604d0a8163cb2fca223efa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733776996244 2024-12-09T20:43:20,617 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f566dd219c4846b79fb822685af6ba05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733776996480 2024-12-09T20:43:20,617 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:20,618 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/A is initiating minor compaction (all files) 2024-12-09T20:43:20,618 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/A in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:20,618 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 99fb21457d75434d9a310c77bcc85a16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733776996544 2024-12-09T20:43:20,618 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/c55432042af94d00922ec36d33b56fd8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e26a01ecd0a741d18fc8c3a827ff6d2b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f93a70f707a849558b51be9a3e5ea4cf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/2de4d5a6b86a401fbf398f1cebde8db8] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=146.1 K 2024-12-09T20:43:20,618 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:20,618 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/c55432042af94d00922ec36d33b56fd8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e26a01ecd0a741d18fc8c3a827ff6d2b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f93a70f707a849558b51be9a3e5ea4cf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/2de4d5a6b86a401fbf398f1cebde8db8] 2024-12-09T20:43:20,618 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d4f0930d38e4ca2ac9c555cf7b543d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733776997787 2024-12-09T20:43:20,618 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting c55432042af94d00922ec36d33b56fd8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733776996244 2024-12-09T20:43:20,619 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e26a01ecd0a741d18fc8c3a827ff6d2b, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733776996424 2024-12-09T20:43:20,619 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting f93a70f707a849558b51be9a3e5ea4cf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733776996544 2024-12-09T20:43:20,619 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de4d5a6b86a401fbf398f1cebde8db8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733776997787 2024-12-09T20:43:20,625 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:20,625 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/910674ab28ea46ffb3805b426880fc97 is 50, key is test_row_0/B:col10/1733776997802/Put/seqid=0 2024-12-09T20:43:20,628 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:20,630 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209549daa47249f4322a97f0f8c8c7f29a9_f4469ee439a16ac91972d18bc523679b store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:20,632 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209549daa47249f4322a97f0f8c8c7f29a9_f4469ee439a16ac91972d18bc523679b, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:20,633 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209549daa47249f4322a97f0f8c8c7f29a9_f4469ee439a16ac91972d18bc523679b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:20,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742175_1351 (size=12139) 2024-12-09T20:43:20,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742176_1352 (size=4469) 2024-12-09T20:43:20,647 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#A#compaction#301 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:20,647 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/abbf767ad55a4be28163dcc56d0bb3c0 is 175, key is test_row_0/A:col10/1733776997802/Put/seqid=0 2024-12-09T20:43:20,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742177_1353 (size=31093) 2024-12-09T20:43:20,659 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/abbf767ad55a4be28163dcc56d0bb3c0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/abbf767ad55a4be28163dcc56d0bb3c0 2024-12-09T20:43:20,662 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4469ee439a16ac91972d18bc523679b/A of f4469ee439a16ac91972d18bc523679b into abbf767ad55a4be28163dcc56d0bb3c0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:20,662 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:20,662 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/A, priority=12, startTime=1733777000616; duration=0sec 2024-12-09T20:43:20,662 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:20,662 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:A 2024-12-09T20:43:20,663 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:20,665 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:20,665 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/C is initiating minor compaction (all files) 2024-12-09T20:43:20,665 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/C in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:20,665 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6a99e18bf4734257bfc6028097f1606f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4c0d624b6e1e48528dc6eb32f000c0f6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/57f01127f71e40349cf3dcaa97a52cf9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/3ca4f2d1867e400eaaa23b5e801b55dd] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=46.9 K 2024-12-09T20:43:20,665 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a99e18bf4734257bfc6028097f1606f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733776996244 2024-12-09T20:43:20,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c0d624b6e1e48528dc6eb32f000c0f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733776996480 2024-12-09T20:43:20,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f01127f71e40349cf3dcaa97a52cf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733776996544 2024-12-09T20:43:20,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ca4f2d1867e400eaaa23b5e801b55dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733776997787 2024-12-09T20:43:20,675 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#C#compaction#302 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:20,676 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/5d9ed05c658a4575b630c019c5ac8f9e is 50, key is test_row_0/C:col10/1733776997802/Put/seqid=0 2024-12-09T20:43:20,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742178_1354 (size=12139) 2024-12-09T20:43:20,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T20:43:20,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:20,688 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:43:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209774057ff5bfd449baeb17b6469817825_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733776998972/Put/seqid=0 2024-12-09T20:43:20,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742179_1355 (size=12154) 2024-12-09T20:43:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:20,701 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209774057ff5bfd449baeb17b6469817825_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209774057ff5bfd449baeb17b6469817825_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T20:43:20,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/37f8c0fbd2254987ad694b4baccec24d, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:20,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/37f8c0fbd2254987ad694b4baccec24d is 175, key is test_row_0/A:col10/1733776998972/Put/seqid=0 2024-12-09T20:43:20,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742180_1356 (size=30955) 2024-12-09T20:43:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T20:43:21,047 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/910674ab28ea46ffb3805b426880fc97 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/910674ab28ea46ffb3805b426880fc97 2024-12-09T20:43:21,056 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4469ee439a16ac91972d18bc523679b/B of f4469ee439a16ac91972d18bc523679b into 910674ab28ea46ffb3805b426880fc97(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:21,056 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:21,056 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/B, priority=12, startTime=1733777000616; duration=0sec 2024-12-09T20:43:21,056 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:21,056 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:B 2024-12-09T20:43:21,086 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/5d9ed05c658a4575b630c019c5ac8f9e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5d9ed05c658a4575b630c019c5ac8f9e 2024-12-09T20:43:21,089 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4469ee439a16ac91972d18bc523679b/C of f4469ee439a16ac91972d18bc523679b into 5d9ed05c658a4575b630c019c5ac8f9e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:21,089 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:21,089 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/C, priority=12, startTime=1733777000616; duration=0sec 2024-12-09T20:43:21,089 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:21,089 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:C 2024-12-09T20:43:21,107 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/37f8c0fbd2254987ad694b4baccec24d 2024-12-09T20:43:21,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/afdefbab1ee4447a89739101579dd825 is 50, key is test_row_0/B:col10/1733776998972/Put/seqid=0 2024-12-09T20:43:21,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742181_1357 (size=12001) 2024-12-09T20:43:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:21,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:21,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777061163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777061164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777061165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777061167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777061168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777061271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777061281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777061281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777061281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777061281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777061481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777061491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777061491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777061491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777061491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,516 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/afdefbab1ee4447a89739101579dd825 2024-12-09T20:43:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T20:43:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/83d46b6d7ab842ceb15f5b95a704dfef is 50, key is test_row_0/C:col10/1733776998972/Put/seqid=0 2024-12-09T20:43:21,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742182_1358 (size=12001) 2024-12-09T20:43:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777061791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777061802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777061802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777061802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777061802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:21,927 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/83d46b6d7ab842ceb15f5b95a704dfef 2024-12-09T20:43:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/37f8c0fbd2254987ad694b4baccec24d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/37f8c0fbd2254987ad694b4baccec24d 2024-12-09T20:43:21,934 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/37f8c0fbd2254987ad694b4baccec24d, entries=150, sequenceid=113, filesize=30.2 K 2024-12-09T20:43:21,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/afdefbab1ee4447a89739101579dd825 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/afdefbab1ee4447a89739101579dd825 2024-12-09T20:43:21,937 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/afdefbab1ee4447a89739101579dd825, entries=150, sequenceid=113, filesize=11.7 K 2024-12-09T20:43:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/83d46b6d7ab842ceb15f5b95a704dfef as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/83d46b6d7ab842ceb15f5b95a704dfef 2024-12-09T20:43:21,941 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/83d46b6d7ab842ceb15f5b95a704dfef, entries=150, sequenceid=113, filesize=11.7 K 2024-12-09T20:43:21,949 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f4469ee439a16ac91972d18bc523679b in 1260ms, sequenceid=113, compaction requested=false 2024-12-09T20:43:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-09T20:43:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-09T20:43:21,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-09T20:43:21,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-12-09T20:43:21,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.5710 sec 2024-12-09T20:43:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:22,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:43:22,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:22,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:22,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:22,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:22,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:22,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:22,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090b0efc5254fe4f1ea616410e9e872706_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777001154/Put/seqid=0 2024-12-09T20:43:22,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777062326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777062327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777062328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742183_1359 (size=12254) 2024-12-09T20:43:22,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777062329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777062330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777062441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777062441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777062441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777062441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777062441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T20:43:22,531 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:22,532 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-09T20:43:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T20:43:22,534 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:22,534 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:22,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T20:43:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777062651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777062651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777062651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777062651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777062651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:22,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:22,738 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:22,740 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090b0efc5254fe4f1ea616410e9e872706_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090b0efc5254fe4f1ea616410e9e872706_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:22,742 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/739097874fec422aa33fa861fb9609a9, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:22,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/739097874fec422aa33fa861fb9609a9 is 175, key is test_row_0/A:col10/1733777001154/Put/seqid=0 2024-12-09T20:43:22,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742184_1360 (size=31055) 2024-12-09T20:43:22,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:22,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:22,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:22,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T20:43:22,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777062961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777062963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777062963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777062963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777062963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:22,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:22,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:22,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:22,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:22,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:22,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,147 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/739097874fec422aa33fa861fb9609a9 2024-12-09T20:43:23,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/679a45832668454d854dcf1a92ff2217 is 50, key is test_row_0/B:col10/1733777001154/Put/seqid=0 2024-12-09T20:43:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T20:43:23,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742185_1361 (size=12101) 2024-12-09T20:43:23,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:23,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:23,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:23,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:23,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777063471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:23,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777063471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:23,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777063471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:23,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777063481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:23,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777063481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:23,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/679a45832668454d854dcf1a92ff2217 2024-12-09T20:43:23,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/4653d563d86645b483b0b9ff579935b5 is 50, key is test_row_0/C:col10/1733777001154/Put/seqid=0 2024-12-09T20:43:23,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742186_1362 (size=12101) 2024-12-09T20:43:23,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T20:43:23,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:23,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:23,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/4653d563d86645b483b0b9ff579935b5 2024-12-09T20:43:23,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/739097874fec422aa33fa861fb9609a9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/739097874fec422aa33fa861fb9609a9 2024-12-09T20:43:23,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/739097874fec422aa33fa861fb9609a9, entries=150, sequenceid=135, filesize=30.3 K 2024-12-09T20:43:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/679a45832668454d854dcf1a92ff2217 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/679a45832668454d854dcf1a92ff2217 2024-12-09T20:43:24,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/679a45832668454d854dcf1a92ff2217, entries=150, sequenceid=135, filesize=11.8 K 2024-12-09T20:43:24,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/4653d563d86645b483b0b9ff579935b5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4653d563d86645b483b0b9ff579935b5 2024-12-09T20:43:24,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4653d563d86645b483b0b9ff579935b5, entries=150, sequenceid=135, filesize=11.8 K 2024-12-09T20:43:24,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f4469ee439a16ac91972d18bc523679b in 1691ms, sequenceid=135, compaction requested=true 2024-12-09T20:43:24,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:24,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:24,007 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:24,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:24,007 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:24,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:24,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:24,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:24,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:24,007 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93103 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:24,007 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/A is initiating minor compaction (all files) 2024-12-09T20:43:24,007 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/A in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:24,008 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/abbf767ad55a4be28163dcc56d0bb3c0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/37f8c0fbd2254987ad694b4baccec24d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/739097874fec422aa33fa861fb9609a9] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=90.9 K 2024-12-09T20:43:24,008 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:24,008 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/abbf767ad55a4be28163dcc56d0bb3c0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/37f8c0fbd2254987ad694b4baccec24d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/739097874fec422aa33fa861fb9609a9] 2024-12-09T20:43:24,008 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:24,008 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting abbf767ad55a4be28163dcc56d0bb3c0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733776997787 2024-12-09T20:43:24,008 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/B is initiating minor compaction (all files) 2024-12-09T20:43:24,008 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/B in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:24,008 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/910674ab28ea46ffb3805b426880fc97, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/afdefbab1ee4447a89739101579dd825, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/679a45832668454d854dcf1a92ff2217] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=35.4 K 2024-12-09T20:43:24,008 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37f8c0fbd2254987ad694b4baccec24d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733776998972 2024-12-09T20:43:24,008 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 910674ab28ea46ffb3805b426880fc97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733776997787 2024-12-09T20:43:24,008 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting afdefbab1ee4447a89739101579dd825, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733776998972 2024-12-09T20:43:24,008 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 739097874fec422aa33fa861fb9609a9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733777001154 2024-12-09T20:43:24,009 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 679a45832668454d854dcf1a92ff2217, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733777001154 2024-12-09T20:43:24,015 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:24,016 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2bc71931f4004467b74aa60cd4d1344a is 50, key is test_row_0/B:col10/1733777001154/Put/seqid=0 2024-12-09T20:43:24,017 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:24,019 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412093a5d434b1dab4c829efc9880f3a8b076_f4469ee439a16ac91972d18bc523679b store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:24,020 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412093a5d434b1dab4c829efc9880f3a8b076_f4469ee439a16ac91972d18bc523679b, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:24,020 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093a5d434b1dab4c829efc9880f3a8b076_f4469ee439a16ac91972d18bc523679b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:24,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742187_1363 (size=12341) 2024-12-09T20:43:24,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742188_1364 (size=4469) 2024-12-09T20:43:24,041 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#A#compaction#310 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:24,042 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/41549d18af2e40d8ab67740fd37f7f23 is 175, key is test_row_0/A:col10/1733777001154/Put/seqid=0 2024-12-09T20:43:24,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742189_1365 (size=31295) 2024-12-09T20:43:24,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T20:43:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:24,066 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:43:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:24,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120956e41c9ea0c34e25a9a6f5f3dd3c6093_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777002325/Put/seqid=0 2024-12-09T20:43:24,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742190_1366 (size=12304) 2024-12-09T20:43:24,431 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2bc71931f4004467b74aa60cd4d1344a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2bc71931f4004467b74aa60cd4d1344a 2024-12-09T20:43:24,436 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/B of f4469ee439a16ac91972d18bc523679b into 2bc71931f4004467b74aa60cd4d1344a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:24,436 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:24,436 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/B, priority=13, startTime=1733777004007; duration=0sec 2024-12-09T20:43:24,436 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:24,436 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:B 2024-12-09T20:43:24,436 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:24,437 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:24,437 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/C is initiating minor compaction (all files) 2024-12-09T20:43:24,437 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/C in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:24,437 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5d9ed05c658a4575b630c019c5ac8f9e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/83d46b6d7ab842ceb15f5b95a704dfef, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4653d563d86645b483b0b9ff579935b5] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=35.4 K 2024-12-09T20:43:24,438 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d9ed05c658a4575b630c019c5ac8f9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733776997787 2024-12-09T20:43:24,438 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d46b6d7ab842ceb15f5b95a704dfef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733776998972 2024-12-09T20:43:24,438 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 4653d563d86645b483b0b9ff579935b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733777001154 2024-12-09T20:43:24,456 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:24,457 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/d7d4df8962254269a000392fa3a595c6 is 50, key is test_row_0/C:col10/1733777001154/Put/seqid=0 2024-12-09T20:43:24,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742191_1367 (size=12341) 2024-12-09T20:43:24,467 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/41549d18af2e40d8ab67740fd37f7f23 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/41549d18af2e40d8ab67740fd37f7f23 2024-12-09T20:43:24,470 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/d7d4df8962254269a000392fa3a595c6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d7d4df8962254269a000392fa3a595c6 2024-12-09T20:43:24,473 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/A of f4469ee439a16ac91972d18bc523679b into 41549d18af2e40d8ab67740fd37f7f23(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:24,473 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:24,473 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/A, priority=13, startTime=1733777004006; duration=0sec 2024-12-09T20:43:24,473 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:24,473 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:A 2024-12-09T20:43:24,475 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/C of f4469ee439a16ac91972d18bc523679b into d7d4df8962254269a000392fa3a595c6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:24,475 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:24,475 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/C, priority=13, startTime=1733777004007; duration=0sec 2024-12-09T20:43:24,475 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:24,475 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:C 2024-12-09T20:43:24,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:24,481 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120956e41c9ea0c34e25a9a6f5f3dd3c6093_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120956e41c9ea0c34e25a9a6f5f3dd3c6093_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:24,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/5e493ee58acf420e9626b2ee8d644cfc, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:24,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/5e493ee58acf420e9626b2ee8d644cfc is 175, key is test_row_0/A:col10/1733777002325/Put/seqid=0 2024-12-09T20:43:24,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742192_1368 (size=31105) 2024-12-09T20:43:24,487 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/5e493ee58acf420e9626b2ee8d644cfc 2024-12-09T20:43:24,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2b6c9c29633c435c8a738258734703d1 is 50, key is test_row_0/B:col10/1733777002325/Put/seqid=0 2024-12-09T20:43:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:24,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742193_1369 (size=12151) 2024-12-09T20:43:24,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777064515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777064516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777064516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777064517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777064518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777064631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777064631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777064631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777064631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777064631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T20:43:24,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777064841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777064841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777064841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777064841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777064841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:24,897 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2b6c9c29633c435c8a738258734703d1 2024-12-09T20:43:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/cf3b297c90034764b7eb0c92624b20ff is 50, key is test_row_0/C:col10/1733777002325/Put/seqid=0 2024-12-09T20:43:24,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742194_1370 (size=12151) 2024-12-09T20:43:25,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777065151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777065151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777065152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777065152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777065161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,311 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/cf3b297c90034764b7eb0c92624b20ff 2024-12-09T20:43:25,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/5e493ee58acf420e9626b2ee8d644cfc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/5e493ee58acf420e9626b2ee8d644cfc 2024-12-09T20:43:25,323 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/5e493ee58acf420e9626b2ee8d644cfc, entries=150, sequenceid=147, filesize=30.4 K 2024-12-09T20:43:25,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2b6c9c29633c435c8a738258734703d1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2b6c9c29633c435c8a738258734703d1 2024-12-09T20:43:25,328 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2b6c9c29633c435c8a738258734703d1, entries=150, sequenceid=147, filesize=11.9 K 2024-12-09T20:43:25,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/cf3b297c90034764b7eb0c92624b20ff as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/cf3b297c90034764b7eb0c92624b20ff 2024-12-09T20:43:25,333 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/cf3b297c90034764b7eb0c92624b20ff, entries=150, sequenceid=147, filesize=11.9 K 2024-12-09T20:43:25,334 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f4469ee439a16ac91972d18bc523679b in 1268ms, sequenceid=147, compaction requested=false 2024-12-09T20:43:25,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:25,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:25,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-09T20:43:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-09T20:43:25,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-09T20:43:25,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8010 sec 2024-12-09T20:43:25,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.8050 sec 2024-12-09T20:43:25,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777004506, max timestamp=1733777005666Num uniques 859; 2024-12-09T20:43:25,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777004506, max timestamp=1733777005666Num uniques 859; 2024-12-09T20:43:25,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777004506, max timestamp=1733777005666Num uniques 859; 2024-12-09T20:43:25,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:25,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:25,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:25,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-09T20:43:25,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:25,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:25,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:25,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:25,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:25,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:25,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096c933eec47214de89b8aeaa01b80f901_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777005677/Put/seqid=0 2024-12-09T20:43:25,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742195_1371 (size=12304) 2024-12-09T20:43:25,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733777005704, max timestamp=1733777005723Num uniques 859; 2024-12-09T20:43:25,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-09T20:43:25,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733777005704, max timestamp=1733777005723Num uniques 859; 2024-12-09T20:43:25,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-09T20:43:25,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733777005704, max timestamp=1733777005723Num uniques 859; 2024-12-09T20:43:25,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-09T20:43:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777065751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777065753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777065754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777065756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777065764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777065881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777065881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777065882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777065882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777065882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:25,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:25,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777066101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777066101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777066101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777066102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777066102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,124 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:26,128 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096c933eec47214de89b8aeaa01b80f901_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096c933eec47214de89b8aeaa01b80f901_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:26,129 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e14b3f8df46e421e9fa644e11d10e5e3, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:26,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e14b3f8df46e421e9fa644e11d10e5e3 is 175, key is test_row_0/A:col10/1733777005677/Put/seqid=0 2024-12-09T20:43:26,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742196_1372 (size=31105) 2024-12-09T20:43:26,153 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e14b3f8df46e421e9fa644e11d10e5e3 2024-12-09T20:43:26,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/1526fa63f8cb42f58cbcca722f78d64b is 50, key is test_row_0/B:col10/1733777005677/Put/seqid=0 2024-12-09T20:43:26,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742197_1373 (size=12151) 2024-12-09T20:43:26,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/1526fa63f8cb42f58cbcca722f78d64b 2024-12-09T20:43:26,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/e21bfb4439334144a7cdeae0bce7546f is 50, key is test_row_0/C:col10/1733777005677/Put/seqid=0 2024-12-09T20:43:26,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742198_1374 (size=12151) 2024-12-09T20:43:26,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777066422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777066428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777066429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777066429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777066429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/e21bfb4439334144a7cdeae0bce7546f 2024-12-09T20:43:26,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e14b3f8df46e421e9fa644e11d10e5e3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e14b3f8df46e421e9fa644e11d10e5e3 2024-12-09T20:43:26,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e14b3f8df46e421e9fa644e11d10e5e3, entries=150, sequenceid=176, filesize=30.4 K 2024-12-09T20:43:26,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/1526fa63f8cb42f58cbcca722f78d64b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/1526fa63f8cb42f58cbcca722f78d64b 2024-12-09T20:43:26,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/1526fa63f8cb42f58cbcca722f78d64b, entries=150, sequenceid=176, filesize=11.9 K 2024-12-09T20:43:26,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/e21bfb4439334144a7cdeae0bce7546f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e21bfb4439334144a7cdeae0bce7546f 2024-12-09T20:43:26,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e21bfb4439334144a7cdeae0bce7546f, entries=150, sequenceid=176, filesize=11.9 K 2024-12-09T20:43:26,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for f4469ee439a16ac91972d18bc523679b in 962ms, sequenceid=176, compaction requested=true 2024-12-09T20:43:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:26,639 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:26,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:43:26,639 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:26,640 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:26,640 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/A is initiating minor compaction (all files) 2024-12-09T20:43:26,640 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/A in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:26,640 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/41549d18af2e40d8ab67740fd37f7f23, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/5e493ee58acf420e9626b2ee8d644cfc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e14b3f8df46e421e9fa644e11d10e5e3] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=91.3 K 2024-12-09T20:43:26,640 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:26,641 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/41549d18af2e40d8ab67740fd37f7f23, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/5e493ee58acf420e9626b2ee8d644cfc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e14b3f8df46e421e9fa644e11d10e5e3] 2024-12-09T20:43:26,641 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:26,641 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/B is initiating minor compaction (all files) 2024-12-09T20:43:26,641 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/B in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:26,641 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2bc71931f4004467b74aa60cd4d1344a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2b6c9c29633c435c8a738258734703d1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/1526fa63f8cb42f58cbcca722f78d64b] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=35.8 K 2024-12-09T20:43:26,641 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41549d18af2e40d8ab67740fd37f7f23, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733777001154 2024-12-09T20:43:26,641 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc71931f4004467b74aa60cd4d1344a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733777001154 2024-12-09T20:43:26,642 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b6c9c29633c435c8a738258734703d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733777002324 2024-12-09T20:43:26,642 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e493ee58acf420e9626b2ee8d644cfc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733777002324 2024-12-09T20:43:26,642 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 1526fa63f8cb42f58cbcca722f78d64b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733777005675 2024-12-09T20:43:26,642 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e14b3f8df46e421e9fa644e11d10e5e3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733777005675 2024-12-09T20:43:26,648 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:26,650 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#B#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:26,651 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/bdcc8c3188f541fd97f902f5da670109 is 50, key is test_row_0/B:col10/1733777005677/Put/seqid=0 2024-12-09T20:43:26,652 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412097e29076a6f9542bd994aa1dac60de1ab_f4469ee439a16ac91972d18bc523679b store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:26,653 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412097e29076a6f9542bd994aa1dac60de1ab_f4469ee439a16ac91972d18bc523679b, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:26,654 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097e29076a6f9542bd994aa1dac60de1ab_f4469ee439a16ac91972d18bc523679b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:26,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742200_1376 (size=4469) 2024-12-09T20:43:26,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742199_1375 (size=12493) 2024-12-09T20:43:26,664 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#A#compaction#318 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:26,664 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e1f99a8cd65e40e0b0e9f1e5ccf40f6a is 175, key is test_row_0/A:col10/1733777005677/Put/seqid=0 2024-12-09T20:43:26,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742201_1377 (size=31447) 2024-12-09T20:43:26,675 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/e1f99a8cd65e40e0b0e9f1e5ccf40f6a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e1f99a8cd65e40e0b0e9f1e5ccf40f6a 2024-12-09T20:43:26,682 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/A of f4469ee439a16ac91972d18bc523679b into e1f99a8cd65e40e0b0e9f1e5ccf40f6a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:26,682 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:26,682 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/A, priority=13, startTime=1733777006639; duration=0sec 2024-12-09T20:43:26,682 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:26,682 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:A 2024-12-09T20:43:26,682 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:26,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:26,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/C is initiating minor compaction (all files) 2024-12-09T20:43:26,683 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/C in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:26,683 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d7d4df8962254269a000392fa3a595c6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/cf3b297c90034764b7eb0c92624b20ff, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e21bfb4439334144a7cdeae0bce7546f] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=35.8 K 2024-12-09T20:43:26,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d4df8962254269a000392fa3a595c6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733777001154 2024-12-09T20:43:26,683 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf3b297c90034764b7eb0c92624b20ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733777002324 2024-12-09T20:43:26,684 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e21bfb4439334144a7cdeae0bce7546f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733777005675 2024-12-09T20:43:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T20:43:26,691 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:26,692 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:26,692 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:26,692 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/5acc1c80f74341fa856068bd15da14f1 is 50, key is test_row_0/C:col10/1733777005677/Put/seqid=0 2024-12-09T20:43:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-09T20:43:26,693 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T20:43:26,695 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:26,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:26,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742202_1378 (size=12493) 2024-12-09T20:43:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T20:43:26,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T20:43:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:26,847 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=127.47 KB heapSize=346.31 KB 2024-12-09T20:43:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120919025525e0e347c3b3e13597f4c25d0c_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777005762/Put/seqid=0 2024-12-09T20:43:26,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742203_1379 (size=12304) 2024-12-09T20:43:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:26,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777066955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777066953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777066965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777066965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:26,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777066966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T20:43:27,067 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/bdcc8c3188f541fd97f902f5da670109 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bdcc8c3188f541fd97f902f5da670109 2024-12-09T20:43:27,070 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/B of f4469ee439a16ac91972d18bc523679b into bdcc8c3188f541fd97f902f5da670109(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:27,071 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:27,071 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/B, priority=13, startTime=1733777006639; duration=0sec 2024-12-09T20:43:27,071 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:27,071 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:B 2024-12-09T20:43:27,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777067071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777067071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777067081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777067081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777067081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,103 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/5acc1c80f74341fa856068bd15da14f1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5acc1c80f74341fa856068bd15da14f1 2024-12-09T20:43:27,107 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/C of f4469ee439a16ac91972d18bc523679b into 5acc1c80f74341fa856068bd15da14f1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:27,107 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:27,107 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/C, priority=13, startTime=1733777006639; duration=0sec 2024-12-09T20:43:27,107 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:27,107 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:C 2024-12-09T20:43:27,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:27,277 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120919025525e0e347c3b3e13597f4c25d0c_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120919025525e0e347c3b3e13597f4c25d0c_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:27,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/96495040821e43f0b6d9d22321b8d9cb, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:27,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/96495040821e43f0b6d9d22321b8d9cb is 175, key is test_row_0/A:col10/1733777005762/Put/seqid=0 2024-12-09T20:43:27,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777067281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777067281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742204_1380 (size=31105) 2024-12-09T20:43:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777067291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777067291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777067291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T20:43:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777067591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777067591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777067601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777067601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777067611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:27,686 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/96495040821e43f0b6d9d22321b8d9cb 2024-12-09T20:43:27,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/05174a42df614d278558a307ba97ca32 is 50, key is test_row_0/B:col10/1733777005762/Put/seqid=0 2024-12-09T20:43:27,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742205_1381 (size=12151) 2024-12-09T20:43:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T20:43:28,098 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/05174a42df614d278558a307ba97ca32 2024-12-09T20:43:28,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777068101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:28,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/e23d96266c8d4b4e969ead3d757c3491 is 50, key is test_row_0/C:col10/1733777005762/Put/seqid=0 2024-12-09T20:43:28,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777068112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:28,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:28,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777068112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:28,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:28,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:28,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777068112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:28,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742206_1382 (size=12151) 2024-12-09T20:43:28,122 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/e23d96266c8d4b4e969ead3d757c3491 2024-12-09T20:43:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/96495040821e43f0b6d9d22321b8d9cb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/96495040821e43f0b6d9d22321b8d9cb 2024-12-09T20:43:28,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777068121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:28,130 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/96495040821e43f0b6d9d22321b8d9cb, entries=150, sequenceid=209, filesize=30.4 K 2024-12-09T20:43:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/05174a42df614d278558a307ba97ca32 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/05174a42df614d278558a307ba97ca32 2024-12-09T20:43:28,135 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/05174a42df614d278558a307ba97ca32, entries=150, sequenceid=209, filesize=11.9 K 2024-12-09T20:43:28,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/e23d96266c8d4b4e969ead3d757c3491 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e23d96266c8d4b4e969ead3d757c3491 2024-12-09T20:43:28,140 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e23d96266c8d4b4e969ead3d757c3491, entries=150, sequenceid=209, filesize=11.9 K 2024-12-09T20:43:28,141 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~346.27 KB/354576, currentSize=60.38 KB/61830 for f4469ee439a16ac91972d18bc523679b in 1293ms, sequenceid=209, compaction requested=false 2024-12-09T20:43:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-09T20:43:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-09T20:43:28,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-09T20:43:28,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-12-09T20:43:28,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4530 sec 2024-12-09T20:43:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T20:43:28,841 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:28,841 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-09T20:43:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T20:43:28,843 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:28,844 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:28,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T20:43:28,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T20:43:28,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:28,997 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:43:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:29,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091a58480939f04fac9f580049064e1cac_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777006963/Put/seqid=0 2024-12-09T20:43:29,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742207_1383 (size=12304) 2024-12-09T20:43:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:29,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:29,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777069149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777069150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777069151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777069155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T20:43:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777069156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777069261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777069261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777069261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777069261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777069271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:29,447 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091a58480939f04fac9f580049064e1cac_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091a58480939f04fac9f580049064e1cac_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:29,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/926caf710f2e492983a8f2275637f5d4, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:29,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/926caf710f2e492983a8f2275637f5d4 is 175, key is test_row_0/A:col10/1733777006963/Put/seqid=0 2024-12-09T20:43:29,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742208_1384 (size=31105) 2024-12-09T20:43:29,453 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/926caf710f2e492983a8f2275637f5d4 2024-12-09T20:43:29,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/d5f3ac4388834ee5b7367b8d23e0d8bc is 50, key is test_row_0/B:col10/1733777006963/Put/seqid=0 2024-12-09T20:43:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T20:43:29,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742209_1385 (size=12151) 2024-12-09T20:43:29,474 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/d5f3ac4388834ee5b7367b8d23e0d8bc 2024-12-09T20:43:29,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777069472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777069472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777069472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777069472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777069481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/d06cd967e5c947d994abde0aa5a2de82 is 50, key is test_row_0/C:col10/1733777006963/Put/seqid=0 2024-12-09T20:43:29,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742210_1386 (size=12151) 2024-12-09T20:43:29,509 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/d06cd967e5c947d994abde0aa5a2de82 2024-12-09T20:43:29,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/926caf710f2e492983a8f2275637f5d4 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/926caf710f2e492983a8f2275637f5d4 2024-12-09T20:43:29,527 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/926caf710f2e492983a8f2275637f5d4, entries=150, sequenceid=223, filesize=30.4 K 2024-12-09T20:43:29,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/d5f3ac4388834ee5b7367b8d23e0d8bc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/d5f3ac4388834ee5b7367b8d23e0d8bc 2024-12-09T20:43:29,541 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/d5f3ac4388834ee5b7367b8d23e0d8bc, entries=150, sequenceid=223, filesize=11.9 K 2024-12-09T20:43:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/d06cd967e5c947d994abde0aa5a2de82 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d06cd967e5c947d994abde0aa5a2de82 2024-12-09T20:43:29,547 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d06cd967e5c947d994abde0aa5a2de82, entries=150, sequenceid=223, filesize=11.9 K 2024-12-09T20:43:29,548 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f4469ee439a16ac91972d18bc523679b in 552ms, sequenceid=223, compaction requested=true 2024-12-09T20:43:29,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:29,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:29,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-09T20:43:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-09T20:43:29,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-09T20:43:29,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 708 msec 2024-12-09T20:43:29,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 714 msec 2024-12-09T20:43:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:29,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T20:43:29,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:29,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:29,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:29,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:29,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:29,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:29,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ce745826655c4cc7bc825d3d86ea9f2d_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777009148/Put/seqid=0 2024-12-09T20:43:29,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777069795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777069796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777069798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777069798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777069799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742211_1387 (size=14794) 2024-12-09T20:43:29,809 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,811 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ce745826655c4cc7bc825d3d86ea9f2d_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ce745826655c4cc7bc825d3d86ea9f2d_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:29,812 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/18217abb5af84797b67ad4da2c71d2b0, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:29,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/18217abb5af84797b67ad4da2c71d2b0 is 175, key is test_row_0/A:col10/1733777009148/Put/seqid=0 2024-12-09T20:43:29,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742212_1388 (size=39749) 2024-12-09T20:43:29,827 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/18217abb5af84797b67ad4da2c71d2b0 2024-12-09T20:43:29,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/bae08ce230fc4c64b920b73168b5a682 is 50, key is test_row_0/B:col10/1733777009148/Put/seqid=0 2024-12-09T20:43:29,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742213_1389 (size=12151) 2024-12-09T20:43:29,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777069912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777069912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777069912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777069912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777069921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T20:43:29,983 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:29,983 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-09T20:43:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T20:43:29,985 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:29,985 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:29,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T20:43:30,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777070121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777070121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777070121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777070121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T20:43:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777070141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/bae08ce230fc4c64b920b73168b5a682 2024-12-09T20:43:30,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/fb480993c89d4688803dc4eaf64114c4 is 50, key is test_row_0/C:col10/1733777009148/Put/seqid=0 2024-12-09T20:43:30,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742214_1390 (size=12151) 2024-12-09T20:43:30,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T20:43:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:30,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T20:43:30,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T20:43:30,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:30,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777070441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777070441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777070441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777070441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777070451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:30,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T20:43:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T20:43:30,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/fb480993c89d4688803dc4eaf64114c4 2024-12-09T20:43:30,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/18217abb5af84797b67ad4da2c71d2b0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/18217abb5af84797b67ad4da2c71d2b0 2024-12-09T20:43:30,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/18217abb5af84797b67ad4da2c71d2b0, entries=200, sequenceid=244, filesize=38.8 K 2024-12-09T20:43:30,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/bae08ce230fc4c64b920b73168b5a682 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bae08ce230fc4c64b920b73168b5a682 2024-12-09T20:43:30,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bae08ce230fc4c64b920b73168b5a682, entries=150, sequenceid=244, filesize=11.9 K 2024-12-09T20:43:30,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/fb480993c89d4688803dc4eaf64114c4 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/fb480993c89d4688803dc4eaf64114c4 2024-12-09T20:43:30,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/fb480993c89d4688803dc4eaf64114c4, entries=150, sequenceid=244, filesize=11.9 K 2024-12-09T20:43:30,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f4469ee439a16ac91972d18bc523679b in 879ms, sequenceid=244, compaction requested=true 2024-12-09T20:43:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:30,663 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:30,663 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:30,664 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:30,664 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:30,664 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/A is initiating minor compaction (all files) 2024-12-09T20:43:30,664 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/B is initiating minor compaction (all files) 2024-12-09T20:43:30,664 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/A in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,664 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/B in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,665 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bdcc8c3188f541fd97f902f5da670109, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/05174a42df614d278558a307ba97ca32, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/d5f3ac4388834ee5b7367b8d23e0d8bc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bae08ce230fc4c64b920b73168b5a682] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=47.8 K 2024-12-09T20:43:30,665 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e1f99a8cd65e40e0b0e9f1e5ccf40f6a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/96495040821e43f0b6d9d22321b8d9cb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/926caf710f2e492983a8f2275637f5d4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/18217abb5af84797b67ad4da2c71d2b0] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=130.3 K 2024-12-09T20:43:30,665 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,665 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e1f99a8cd65e40e0b0e9f1e5ccf40f6a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/96495040821e43f0b6d9d22321b8d9cb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/926caf710f2e492983a8f2275637f5d4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/18217abb5af84797b67ad4da2c71d2b0] 2024-12-09T20:43:30,665 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting bdcc8c3188f541fd97f902f5da670109, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733777005675 2024-12-09T20:43:30,665 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f99a8cd65e40e0b0e9f1e5ccf40f6a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733777005675 2024-12-09T20:43:30,665 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 05174a42df614d278558a307ba97ca32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733777005748 2024-12-09T20:43:30,665 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96495040821e43f0b6d9d22321b8d9cb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733777005748 2024-12-09T20:43:30,665 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d5f3ac4388834ee5b7367b8d23e0d8bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733777006949 2024-12-09T20:43:30,665 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 926caf710f2e492983a8f2275637f5d4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733777006949 2024-12-09T20:43:30,666 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting bae08ce230fc4c64b920b73168b5a682, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777009148 2024-12-09T20:43:30,666 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18217abb5af84797b67ad4da2c71d2b0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777009148 2024-12-09T20:43:30,677 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:30,677 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/8b83d84d4b4f4e6ebb39621a5f7ac598 is 50, key is test_row_0/B:col10/1733777009148/Put/seqid=0 2024-12-09T20:43:30,679 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:30,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742215_1391 (size=12629) 2024-12-09T20:43:30,686 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120943f95b5be301492f845a0fbd44ae2dd5_f4469ee439a16ac91972d18bc523679b store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:30,689 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120943f95b5be301492f845a0fbd44ae2dd5_f4469ee439a16ac91972d18bc523679b, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:30,689 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120943f95b5be301492f845a0fbd44ae2dd5_f4469ee439a16ac91972d18bc523679b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:30,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742216_1392 (size=4469) 2024-12-09T20:43:30,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T20:43:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:30,751 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T20:43:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:30,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120952abc6e2f7264fe7ae3919867f8271a7_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777009795/Put/seqid=0 2024-12-09T20:43:30,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742217_1393 (size=12304) 2024-12-09T20:43:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:31,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777071004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733777010993, max timestamp=1733777011012Num uniques 859; 2024-12-09T20:43:31,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:43:31,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733777010993, max timestamp=1733777011012Num uniques 859; 2024-12-09T20:43:31,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733777010993, max timestamp=1733777011012Num uniques 859; 2024-12-09T20:43:31,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:43:31,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:43:31,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777071046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777071055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777071061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777071063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,091 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/8b83d84d4b4f4e6ebb39621a5f7ac598 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8b83d84d4b4f4e6ebb39621a5f7ac598 2024-12-09T20:43:31,096 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#A#compaction#331 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:31,097 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/1191a6ca786b4d4bb08434b038e1731c is 175, key is test_row_0/A:col10/1733777009148/Put/seqid=0 2024-12-09T20:43:31,098 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4469ee439a16ac91972d18bc523679b/B of f4469ee439a16ac91972d18bc523679b into 8b83d84d4b4f4e6ebb39621a5f7ac598(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:31,098 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:31,098 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/B, priority=12, startTime=1733777010663; duration=0sec 2024-12-09T20:43:31,099 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:31,099 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:B 2024-12-09T20:43:31,099 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:31,104 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:31,104 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/C is initiating minor compaction (all files) 2024-12-09T20:43:31,104 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/C in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:31,104 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5acc1c80f74341fa856068bd15da14f1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e23d96266c8d4b4e969ead3d757c3491, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d06cd967e5c947d994abde0aa5a2de82, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/fb480993c89d4688803dc4eaf64114c4] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=47.8 K 2024-12-09T20:43:31,106 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 5acc1c80f74341fa856068bd15da14f1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733777005675 2024-12-09T20:43:31,106 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting e23d96266c8d4b4e969ead3d757c3491, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733777005748 2024-12-09T20:43:31,107 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d06cd967e5c947d994abde0aa5a2de82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733777006949 2024-12-09T20:43:31,107 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting fb480993c89d4688803dc4eaf64114c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777009148 2024-12-09T20:43:31,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742218_1394 (size=31583) 2024-12-09T20:43:31,121 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#C#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:31,121 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/6746aadbb0fb48d1b21017b35b0d8833 is 50, key is test_row_0/C:col10/1733777009148/Put/seqid=0 2024-12-09T20:43:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T20:43:31,126 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/1191a6ca786b4d4bb08434b038e1731c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/1191a6ca786b4d4bb08434b038e1731c 2024-12-09T20:43:31,132 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4469ee439a16ac91972d18bc523679b/A of f4469ee439a16ac91972d18bc523679b into 1191a6ca786b4d4bb08434b038e1731c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:31,132 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:31,132 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/A, priority=12, startTime=1733777010663; duration=0sec 2024-12-09T20:43:31,133 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:31,133 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:A 2024-12-09T20:43:31,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777071122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742219_1395 (size=12629) 2024-12-09T20:43:31,146 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/6746aadbb0fb48d1b21017b35b0d8833 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6746aadbb0fb48d1b21017b35b0d8833 2024-12-09T20:43:31,153 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4469ee439a16ac91972d18bc523679b/C of f4469ee439a16ac91972d18bc523679b into 6746aadbb0fb48d1b21017b35b0d8833(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:31,153 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:31,153 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/C, priority=12, startTime=1733777010663; duration=0sec 2024-12-09T20:43:31,153 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:31,153 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:C 2024-12-09T20:43:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:31,168 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120952abc6e2f7264fe7ae3919867f8271a7_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120952abc6e2f7264fe7ae3919867f8271a7_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:31,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/360d4f2068a147118ac83bfe0bf3f123, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:31,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/360d4f2068a147118ac83bfe0bf3f123 is 175, key is test_row_0/A:col10/1733777009795/Put/seqid=0 2024-12-09T20:43:31,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742220_1396 (size=31105) 2024-12-09T20:43:31,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777071171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777071171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777071192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777071192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777071341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777071391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777071391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777071401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777071411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,574 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/360d4f2068a147118ac83bfe0bf3f123 2024-12-09T20:43:31,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/123606b5b78744b59296396ff014b1f7 is 50, key is test_row_0/B:col10/1733777009795/Put/seqid=0 2024-12-09T20:43:31,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742221_1397 (size=12151) 2024-12-09T20:43:31,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777071651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777071701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777071701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777071721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777071731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:31,987 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/123606b5b78744b59296396ff014b1f7 2024-12-09T20:43:31,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/1265e8dbfcea47e0b39d0efcc7490e61 is 50, key is test_row_0/C:col10/1733777009795/Put/seqid=0 2024-12-09T20:43:31,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742222_1398 (size=12151) 2024-12-09T20:43:31,999 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/1265e8dbfcea47e0b39d0efcc7490e61 2024-12-09T20:43:32,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/360d4f2068a147118ac83bfe0bf3f123 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/360d4f2068a147118ac83bfe0bf3f123 2024-12-09T20:43:32,008 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/360d4f2068a147118ac83bfe0bf3f123, entries=150, sequenceid=255, filesize=30.4 K 2024-12-09T20:43:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/123606b5b78744b59296396ff014b1f7 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/123606b5b78744b59296396ff014b1f7 2024-12-09T20:43:32,012 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/123606b5b78744b59296396ff014b1f7, entries=150, sequenceid=255, filesize=11.9 K 2024-12-09T20:43:32,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/1265e8dbfcea47e0b39d0efcc7490e61 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/1265e8dbfcea47e0b39d0efcc7490e61 2024-12-09T20:43:32,018 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/1265e8dbfcea47e0b39d0efcc7490e61, entries=150, sequenceid=255, filesize=11.9 K 2024-12-09T20:43:32,019 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f4469ee439a16ac91972d18bc523679b in 1268ms, sequenceid=255, compaction requested=false 2024-12-09T20:43:32,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:32,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-09T20:43:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-09T20:43:32,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-09T20:43:32,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0340 sec 2024-12-09T20:43:32,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.0390 sec 2024-12-09T20:43:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T20:43:32,131 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:32,131 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-09T20:43:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T20:43:32,133 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:32,134 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:32,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:32,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-09T20:43:32,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:32,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:32,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:32,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:32,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:32,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:32,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098383bf7090d3470ba664114f37092f0a_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777011061/Put/seqid=0 2024-12-09T20:43:32,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742223_1399 (size=14994) 2024-12-09T20:43:32,188 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:32,193 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098383bf7090d3470ba664114f37092f0a_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098383bf7090d3470ba664114f37092f0a_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:32,194 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/d050b29c826c4758a20d47dc04301c66, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:32,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/d050b29c826c4758a20d47dc04301c66 is 175, key is test_row_0/A:col10/1733777011061/Put/seqid=0 2024-12-09T20:43:32,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742224_1400 (size=39949) 2024-12-09T20:43:32,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777072211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:32,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777072211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777072216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T20:43:32,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777072241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:32,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777072251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:32,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T20:43:32,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:32,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:32,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:32,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777072331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:32,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T20:43:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:32,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T20:43:32,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777072541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:32,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:32,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T20:43:32,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:32,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:32,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:32,599 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/d050b29c826c4758a20d47dc04301c66 2024-12-09T20:43:32,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/3ea9a6c6f6c941c5a294b458f7daa84c is 50, key is test_row_0/B:col10/1733777011061/Put/seqid=0 2024-12-09T20:43:32,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742225_1401 (size=12301) 2024-12-09T20:43:32,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/3ea9a6c6f6c941c5a294b458f7daa84c 2024-12-09T20:43:32,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/b87983569a754736a9eb97b594d4e4af is 50, key is test_row_0/C:col10/1733777011061/Put/seqid=0 2024-12-09T20:43:32,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742226_1402 (size=12301) 2024-12-09T20:43:32,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/b87983569a754736a9eb97b594d4e4af 2024-12-09T20:43:32,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/d050b29c826c4758a20d47dc04301c66 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d050b29c826c4758a20d47dc04301c66 2024-12-09T20:43:32,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d050b29c826c4758a20d47dc04301c66, entries=200, sequenceid=293, filesize=39.0 K 2024-12-09T20:43:32,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/3ea9a6c6f6c941c5a294b458f7daa84c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/3ea9a6c6f6c941c5a294b458f7daa84c 2024-12-09T20:43:32,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/3ea9a6c6f6c941c5a294b458f7daa84c, entries=150, sequenceid=293, filesize=12.0 K 2024-12-09T20:43:32,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/b87983569a754736a9eb97b594d4e4af as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/b87983569a754736a9eb97b594d4e4af 2024-12-09T20:43:32,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/b87983569a754736a9eb97b594d4e4af, entries=150, sequenceid=293, filesize=12.0 K 2024-12-09T20:43:32,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for f4469ee439a16ac91972d18bc523679b in 554ms, sequenceid=293, compaction requested=true 2024-12-09T20:43:32,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:32,721 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:32,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:32,722 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:32,722 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/A is initiating minor compaction (all files) 2024-12-09T20:43:32,722 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/A in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,723 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/1191a6ca786b4d4bb08434b038e1731c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/360d4f2068a147118ac83bfe0bf3f123, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d050b29c826c4758a20d47dc04301c66] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=100.2 K 2024-12-09T20:43:32,723 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,723 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/1191a6ca786b4d4bb08434b038e1731c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/360d4f2068a147118ac83bfe0bf3f123, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d050b29c826c4758a20d47dc04301c66] 2024-12-09T20:43:32,723 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1191a6ca786b4d4bb08434b038e1731c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777009148 2024-12-09T20:43:32,723 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 360d4f2068a147118ac83bfe0bf3f123, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733777009795 2024-12-09T20:43:32,724 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d050b29c826c4758a20d47dc04301c66, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733777011043 2024-12-09T20:43:32,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:32,724 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:32,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:32,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:32,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:32,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:32,727 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:32,727 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/B is initiating minor compaction (all files) 2024-12-09T20:43:32,727 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/B in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,727 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8b83d84d4b4f4e6ebb39621a5f7ac598, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/123606b5b78744b59296396ff014b1f7, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/3ea9a6c6f6c941c5a294b458f7daa84c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=36.2 K 2024-12-09T20:43:32,728 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b83d84d4b4f4e6ebb39621a5f7ac598, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777009148 2024-12-09T20:43:32,729 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 123606b5b78744b59296396ff014b1f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733777009795 2024-12-09T20:43:32,729 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ea9a6c6f6c941c5a294b458f7daa84c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733777011043 2024-12-09T20:43:32,745 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:32,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T20:43:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,749 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:43:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T20:43:32,767 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:32,767 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/42c4a35677d64d11b7fe80d8c69970d0 is 50, key is test_row_0/B:col10/1733777011061/Put/seqid=0 2024-12-09T20:43:32,777 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412092cd1837d9b554a82b9afb905eb0fd5fd_f4469ee439a16ac91972d18bc523679b store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:32,782 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412092cd1837d9b554a82b9afb905eb0fd5fd_f4469ee439a16ac91972d18bc523679b, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:32,782 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092cd1837d9b554a82b9afb905eb0fd5fd_f4469ee439a16ac91972d18bc523679b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:32,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a2409c679181498d93d9665be8d700e7_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777012203/Put/seqid=0 2024-12-09T20:43:32,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742227_1403 (size=12881) 2024-12-09T20:43:32,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742228_1404 (size=4469) 2024-12-09T20:43:32,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:32,901 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#A#compaction#339 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:32,901 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/bdd1e02292b748d4b29515cc035af9df is 175, key is test_row_0/A:col10/1733777011061/Put/seqid=0 2024-12-09T20:43:32,903 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/42c4a35677d64d11b7fe80d8c69970d0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/42c4a35677d64d11b7fe80d8c69970d0 2024-12-09T20:43:32,908 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/B of f4469ee439a16ac91972d18bc523679b into 42c4a35677d64d11b7fe80d8c69970d0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:32,908 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:32,909 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/B, priority=13, startTime=1733777012724; duration=0sec 2024-12-09T20:43:32,909 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:32,909 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:B 2024-12-09T20:43:32,909 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:32,910 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:32,910 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/C is initiating minor compaction (all files) 2024-12-09T20:43:32,910 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/C in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:32,910 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6746aadbb0fb48d1b21017b35b0d8833, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/1265e8dbfcea47e0b39d0efcc7490e61, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/b87983569a754736a9eb97b594d4e4af] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=36.2 K 2024-12-09T20:43:32,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742229_1405 (size=12454) 2024-12-09T20:43:32,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:32,913 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6746aadbb0fb48d1b21017b35b0d8833, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777009148 2024-12-09T20:43:32,914 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 1265e8dbfcea47e0b39d0efcc7490e61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733777009795 2024-12-09T20:43:32,915 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b87983569a754736a9eb97b594d4e4af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733777011043 2024-12-09T20:43:32,917 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a2409c679181498d93d9665be8d700e7_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a2409c679181498d93d9665be8d700e7_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:32,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/982afab06f3949d6ade2699dc806e0a5, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:32,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/982afab06f3949d6ade2699dc806e0a5 is 175, key is test_row_0/A:col10/1733777012203/Put/seqid=0 2024-12-09T20:43:32,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742230_1406 (size=31835) 2024-12-09T20:43:32,950 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:32,951 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/59f254cc883c472c80073b599c3e8367 is 50, key is test_row_0/C:col10/1733777011061/Put/seqid=0 2024-12-09T20:43:32,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742231_1407 (size=31255) 2024-12-09T20:43:32,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742232_1408 (size=12881) 2024-12-09T20:43:33,001 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/59f254cc883c472c80073b599c3e8367 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/59f254cc883c472c80073b599c3e8367 2024-12-09T20:43:33,006 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/C of f4469ee439a16ac91972d18bc523679b into 59f254cc883c472c80073b599c3e8367(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:33,006 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:33,006 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/C, priority=13, startTime=1733777012724; duration=0sec 2024-12-09T20:43:33,006 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:33,006 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:C 2024-12-09T20:43:33,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777013008, max timestamp=1733777013048Num uniques 859; 2024-12-09T20:43:33,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:33,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777013008, max timestamp=1733777013048Num uniques 859; 2024-12-09T20:43:33,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:33,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777013008, max timestamp=1733777013048Num uniques 859; 2024-12-09T20:43:33,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:33,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:33,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777073164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:33,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777073221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:33,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777073221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:33,229 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in f4469ee439a16ac91972d18bc523679b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,230 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in f4469ee439a16ac91972d18bc523679b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:33,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777073251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:33,256 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in f4469ee439a16ac91972d18bc523679b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:33,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T20:43:33,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777073271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:33,283 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in f4469ee439a16ac91972d18bc523679b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777073281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:33,339 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/bdd1e02292b748d4b29515cc035af9df as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/bdd1e02292b748d4b29515cc035af9df 2024-12-09T20:43:33,343 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/A of f4469ee439a16ac91972d18bc523679b into bdd1e02292b748d4b29515cc035af9df(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:33,343 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:33,343 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/A, priority=13, startTime=1733777012721; duration=0sec 2024-12-09T20:43:33,343 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:33,343 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:A 2024-12-09T20:43:33,374 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/982afab06f3949d6ade2699dc806e0a5 2024-12-09T20:43:33,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/8c6475aeffbe412f8e3b64c8ba5c442e is 50, key is test_row_0/B:col10/1733777012203/Put/seqid=0 2024-12-09T20:43:33,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742233_1409 (size=12301) 2024-12-09T20:43:33,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777073501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:33,788 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/8c6475aeffbe412f8e3b64c8ba5c442e 2024-12-09T20:43:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/9ee8a01bc66a48338d3104084815793b is 50, key is test_row_0/C:col10/1733777012203/Put/seqid=0 2024-12-09T20:43:33,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742234_1410 (size=12301) 2024-12-09T20:43:33,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777073821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:34,113 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T20:43:34,200 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/9ee8a01bc66a48338d3104084815793b 2024-12-09T20:43:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/982afab06f3949d6ade2699dc806e0a5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/982afab06f3949d6ade2699dc806e0a5 2024-12-09T20:43:34,207 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/982afab06f3949d6ade2699dc806e0a5, entries=150, sequenceid=303, filesize=30.5 K 2024-12-09T20:43:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/8c6475aeffbe412f8e3b64c8ba5c442e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8c6475aeffbe412f8e3b64c8ba5c442e 2024-12-09T20:43:34,212 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8c6475aeffbe412f8e3b64c8ba5c442e, entries=150, sequenceid=303, filesize=12.0 K 2024-12-09T20:43:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/9ee8a01bc66a48338d3104084815793b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/9ee8a01bc66a48338d3104084815793b 2024-12-09T20:43:34,216 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/9ee8a01bc66a48338d3104084815793b, entries=150, sequenceid=303, filesize=12.0 K 2024-12-09T20:43:34,219 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f4469ee439a16ac91972d18bc523679b in 1470ms, sequenceid=303, compaction requested=false 2024-12-09T20:43:34,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:34,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:34,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-09T20:43:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-09T20:43:34,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-09T20:43:34,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0860 sec 2024-12-09T20:43:34,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.0910 sec 2024-12-09T20:43:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T20:43:34,281 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:34,282 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-09T20:43:34,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T20:43:34,284 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:34,285 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:34,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:34,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733777013138, max timestamp=1733777014335Num uniques 1018; 2024-12-09T20:43:34,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:43:34,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733777013138, max timestamp=1733777014335Num uniques 1018; 2024-12-09T20:43:34,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=102.87 KB heapSize=292.20 KB 2024-12-09T20:43:34,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:43:34,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733777013138, max timestamp=1733777014335Num uniques 1018; 2024-12-09T20:43:34,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:43:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:34,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:34,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209889b29af17084864ae0d77b9b52bb312_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777014337/Put/seqid=0 2024-12-09T20:43:34,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742235_1411 (size=17534) 2024-12-09T20:43:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T20:43:34,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777014396, max timestamp=1733777014429Num uniques 859; 2024-12-09T20:43:34,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:34,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777014396, max timestamp=1733777014429Num uniques 859; 2024-12-09T20:43:34,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:34,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777014396, max timestamp=1733777014429Num uniques 859; 2024-12-09T20:43:34,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:34,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:34,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733777014486, max timestamp=1733777014530Num uniques 1018; 2024-12-09T20:43:34,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:43:34,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733777014486, max timestamp=1733777014530Num uniques 1018; 2024-12-09T20:43:34,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733777014486, max timestamp=1733777014530Num uniques 1018; 2024-12-09T20:43:34,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:43:34,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T20:43:34,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:34,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T20:43:34,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733777014592, max timestamp=1733777014625Num uniques 1027; 2024-12-09T20:43:34,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733777014592, max timestamp=1733777014625Num uniques 1027; 2024-12-09T20:43:34,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T20:43:34,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T20:43:34,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733777014592, max timestamp=1733777014625Num uniques 1027; 2024-12-09T20:43:34,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T20:43:34,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:34,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:34,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733777014712, max timestamp=1733777014750Num uniques 1036; 2024-12-09T20:43:34,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T20:43:34,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733777014712, max timestamp=1733777014750Num uniques 1036; 2024-12-09T20:43:34,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T20:43:34,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733777014712, max timestamp=1733777014750Num uniques 1036; 2024-12-09T20:43:34,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T20:43:34,756 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209889b29af17084864ae0d77b9b52bb312_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209889b29af17084864ae0d77b9b52bb312_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:34,758 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/fa149032c7b941d48879c27671c680b1, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:34,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/fa149032c7b941d48879c27671c680b1 is 175, key is test_row_0/A:col10/1733777014337/Put/seqid=0 2024-12-09T20:43:34,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742236_1412 (size=48639) 2024-12-09T20:43:34,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733777014819, max timestamp=1733777014894Num uniques 1045; 2024-12-09T20:43:34,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733777014819, max timestamp=1733777014894Num uniques 1045; 2024-12-09T20:43:34,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733777014819, max timestamp=1733777014894Num uniques 1045; 2024-12-09T20:43:34,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T20:43:34,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T20:43:34,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T20:43:34,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:34,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T20:43:35,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733777014983, max timestamp=1733777015000Num uniques 1054; 2024-12-09T20:43:35,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T20:43:35,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733777014983, max timestamp=1733777015000Num uniques 1054; 2024-12-09T20:43:35,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T20:43:35,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733777014983, max timestamp=1733777015000Num uniques 1054; 2024-12-09T20:43:35,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T20:43:35,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:35,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:35,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733777015071, max timestamp=1733777015093Num uniques 1013; 2024-12-09T20:43:35,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-09T20:43:35,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733777015071, max timestamp=1733777015093Num uniques 1013; 2024-12-09T20:43:35,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-09T20:43:35,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733777015071, max timestamp=1733777015093Num uniques 1013; 2024-12-09T20:43:35,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-09T20:43:35,162 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/fa149032c7b941d48879c27671c680b1 2024-12-09T20:43:35,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2903fd8134004951831b96b89b90cdec is 50, key is test_row_0/B:col10/1733777014337/Put/seqid=0 2024-12-09T20:43:35,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742237_1413 (size=12301) 2024-12-09T20:43:35,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733777015132, max timestamp=1733777015184Num uniques 1022; 2024-12-09T20:43:35,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733777015132, max timestamp=1733777015184Num uniques 1022; 2024-12-09T20:43:35,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-09T20:43:35,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-09T20:43:35,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733777015132, max timestamp=1733777015184Num uniques 1022; 2024-12-09T20:43:35,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-09T20:43:35,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733777015246, max timestamp=1733777015269Num uniques 1081; 2024-12-09T20:43:35,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-09T20:43:35,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733777015246, max timestamp=1733777015269Num uniques 1081; 2024-12-09T20:43:35,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-09T20:43:35,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733777015246, max timestamp=1733777015269Num uniques 1081; 2024-12-09T20:43:35,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-09T20:43:35,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777075308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733777015284, max timestamp=1733777015317Num uniques 1090; 2024-12-09T20:43:35,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-09T20:43:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733777015284, max timestamp=1733777015317Num uniques 1090; 2024-12-09T20:43:35,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-09T20:43:35,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733777015284, max timestamp=1733777015317Num uniques 1090; 2024-12-09T20:43:35,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-09T20:43:35,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733777015341, max timestamp=1733777015354Num uniques 1049; 2024-12-09T20:43:35,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-09T20:43:35,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733777015341, max timestamp=1733777015354Num uniques 1049; 2024-12-09T20:43:35,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733777015341, max timestamp=1733777015354Num uniques 1049; 2024-12-09T20:43:35,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-09T20:43:35,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-09T20:43:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733777015377, max timestamp=1733777015392Num uniques 1108; 2024-12-09T20:43:35,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-12-09T20:43:35,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733777015377, max timestamp=1733777015392Num uniques 1108; 2024-12-09T20:43:35,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-12-09T20:43:35,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733777015377, max timestamp=1733777015392Num uniques 1108; 2024-12-09T20:43:35,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-12-09T20:43:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T20:43:35,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777075426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733777015417, max timestamp=1733777015433Num uniques 1117; 2024-12-09T20:43:35,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-12-09T20:43:35,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733777015417, max timestamp=1733777015433Num uniques 1117; 2024-12-09T20:43:35,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-12-09T20:43:35,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733777015417, max timestamp=1733777015433Num uniques 1117; 2024-12-09T20:43:35,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-12-09T20:43:35,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1733777015432, max timestamp=1733777015474Num uniques 1076; 2024-12-09T20:43:35,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=14926, suffix heap size=139872,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=326 2024-12-09T20:43:35,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1733777015432, max timestamp=1733777015474Num uniques 1076; 2024-12-09T20:43:35,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=14926, suffix heap size=139872,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=326 2024-12-09T20:43:35,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1733777015432, max timestamp=1733777015474Num uniques 1076; 2024-12-09T20:43:35,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=14926, suffix heap size=139872,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=326 2024-12-09T20:43:35,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777075508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777075510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777075512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777075516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777075541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2903fd8134004951831b96b89b90cdec 2024-12-09T20:43:35,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/71afb51eb0ab439381d106ecc852eec9 is 50, key is test_row_0/C:col10/1733777014337/Put/seqid=0 2024-12-09T20:43:35,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742238_1414 (size=12301) 2024-12-09T20:43:35,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777075631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777075631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777075631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777075641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:35,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:35,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777075751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:35,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:35,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777075841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777075841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777075841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777075861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:35,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:35,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:35,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:35,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:35,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/71afb51eb0ab439381d106ecc852eec9 2024-12-09T20:43:35,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/fa149032c7b941d48879c27671c680b1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/fa149032c7b941d48879c27671c680b1 2024-12-09T20:43:35,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/fa149032c7b941d48879c27671c680b1, entries=250, sequenceid=345, filesize=47.5 K 2024-12-09T20:43:35,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2903fd8134004951831b96b89b90cdec as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2903fd8134004951831b96b89b90cdec 2024-12-09T20:43:36,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2903fd8134004951831b96b89b90cdec, entries=150, sequenceid=345, filesize=12.0 K 2024-12-09T20:43:36,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/71afb51eb0ab439381d106ecc852eec9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/71afb51eb0ab439381d106ecc852eec9 2024-12-09T20:43:36,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/71afb51eb0ab439381d106ecc852eec9, entries=150, sequenceid=345, filesize=12.0 K 2024-12-09T20:43:36,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for f4469ee439a16ac91972d18bc523679b in 1668ms, sequenceid=345, compaction requested=true 2024-12-09T20:43:36,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:36,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:36,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:36,006 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:36,006 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:36,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:36,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:36,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4469ee439a16ac91972d18bc523679b:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:36,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:36,007 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:36,007 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/B is initiating minor compaction (all files) 2024-12-09T20:43:36,007 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/B in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,008 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/42c4a35677d64d11b7fe80d8c69970d0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8c6475aeffbe412f8e3b64c8ba5c442e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2903fd8134004951831b96b89b90cdec] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=36.6 K 2024-12-09T20:43:36,008 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111729 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:36,008 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/A is initiating minor compaction (all files) 2024-12-09T20:43:36,008 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/A in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,008 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 42c4a35677d64d11b7fe80d8c69970d0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733777011043 2024-12-09T20:43:36,008 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/bdd1e02292b748d4b29515cc035af9df, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/982afab06f3949d6ade2699dc806e0a5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/fa149032c7b941d48879c27671c680b1] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=109.1 K 2024-12-09T20:43:36,008 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,008 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/bdd1e02292b748d4b29515cc035af9df, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/982afab06f3949d6ade2699dc806e0a5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/fa149032c7b941d48879c27671c680b1] 2024-12-09T20:43:36,008 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c6475aeffbe412f8e3b64c8ba5c442e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733777012192 2024-12-09T20:43:36,008 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdd1e02292b748d4b29515cc035af9df, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733777011043 2024-12-09T20:43:36,009 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2903fd8134004951831b96b89b90cdec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733777013162 2024-12-09T20:43:36,011 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 982afab06f3949d6ade2699dc806e0a5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733777012192 2024-12-09T20:43:36,011 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa149032c7b941d48879c27671c680b1, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733777013138 2024-12-09T20:43:36,019 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:36,020 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/5b601f5eb2b54277b3622d6e6186fe11 is 50, key is test_row_0/B:col10/1733777014337/Put/seqid=0 2024-12-09T20:43:36,029 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:36,031 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209d2e6734755e7476ea46cd257c8c8282c_f4469ee439a16ac91972d18bc523679b store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:36,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742239_1415 (size=12983) 2024-12-09T20:43:36,033 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209d2e6734755e7476ea46cd257c8c8282c_f4469ee439a16ac91972d18bc523679b, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:36,033 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d2e6734755e7476ea46cd257c8c8282c_f4469ee439a16ac91972d18bc523679b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:36,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742240_1416 (size=4469) 2024-12-09T20:43:36,038 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/5b601f5eb2b54277b3622d6e6186fe11 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/5b601f5eb2b54277b3622d6e6186fe11 2024-12-09T20:43:36,038 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#A#compaction#349 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:36,039 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/f3787792b6834c2491fbae2779f038c6 is 175, key is test_row_0/A:col10/1733777014337/Put/seqid=0 2024-12-09T20:43:36,043 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/B of f4469ee439a16ac91972d18bc523679b into 5b601f5eb2b54277b3622d6e6186fe11(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:36,043 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:36,043 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/B, priority=13, startTime=1733777016006; duration=0sec 2024-12-09T20:43:36,043 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:36,043 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:B 2024-12-09T20:43:36,043 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:36,044 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:36,044 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): f4469ee439a16ac91972d18bc523679b/C is initiating minor compaction (all files) 2024-12-09T20:43:36,044 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4469ee439a16ac91972d18bc523679b/C in TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,044 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/59f254cc883c472c80073b599c3e8367, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/9ee8a01bc66a48338d3104084815793b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/71afb51eb0ab439381d106ecc852eec9] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp, totalSize=36.6 K 2024-12-09T20:43:36,044 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 59f254cc883c472c80073b599c3e8367, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733777011043 2024-12-09T20:43:36,045 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ee8a01bc66a48338d3104084815793b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733777012192 2024-12-09T20:43:36,045 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 71afb51eb0ab439381d106ecc852eec9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733777013162 2024-12-09T20:43:36,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742241_1417 (size=31937) 2024-12-09T20:43:36,056 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4469ee439a16ac91972d18bc523679b#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:36,056 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/96d982b727b14be09c820f476e2bf978 is 50, key is test_row_0/C:col10/1733777014337/Put/seqid=0 2024-12-09T20:43:36,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742242_1418 (size=12983) 2024-12-09T20:43:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:36,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=161.02 KB heapSize=449.72 KB 2024-12-09T20:43:36,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:36,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:43:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:43:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:43:36,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a58e3d2e4f6c484b83bf28422a298ab6_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777015509/Put/seqid=0 2024-12-09T20:43:36,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742243_1419 (size=12454) 2024-12-09T20:43:36,094 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:36,096 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a58e3d2e4f6c484b83bf28422a298ab6_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a58e3d2e4f6c484b83bf28422a298ab6_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:36,097 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/47b1af39a9c3499aa068196059b81e8b, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:36,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/47b1af39a9c3499aa068196059b81e8b is 175, key is test_row_0/A:col10/1733777015509/Put/seqid=0 2024-12-09T20:43:36,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742244_1420 (size=31255) 2024-12-09T20:43:36,103 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=607, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/47b1af39a9c3499aa068196059b81e8b 2024-12-09T20:43:36,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2ea0865206014ff790aad8a423a32b21 is 50, key is test_row_0/B:col10/1733777015509/Put/seqid=0 2024-12-09T20:43:36,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777076107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742245_1421 (size=12301) 2024-12-09T20:43:36,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777076152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777076152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777076161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777076181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777076220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:36,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:36,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,329 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-12-09T20:43:36,329 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:36,329 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,329 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:36,329 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-12-09T20:43:36,329 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:36,329 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,330 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:36,335 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-12-09T20:43:36,335 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:36,335 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,335 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:36,336 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-12-09T20:43:36,336 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:36,336 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,336 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:36,337 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-12-09T20:43:36,337 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:36,337 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,337 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:36,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T20:43:36,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:36,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:36,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777076441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,458 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/f3787792b6834c2491fbae2779f038c6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f3787792b6834c2491fbae2779f038c6 2024-12-09T20:43:36,461 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/A of f4469ee439a16ac91972d18bc523679b into f3787792b6834c2491fbae2779f038c6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:36,461 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:36,461 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/A, priority=13, startTime=1733777016006; duration=0sec 2024-12-09T20:43:36,462 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:36,462 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:A 2024-12-09T20:43:36,464 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/96d982b727b14be09c820f476e2bf978 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/96d982b727b14be09c820f476e2bf978 2024-12-09T20:43:36,467 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4469ee439a16ac91972d18bc523679b/C of f4469ee439a16ac91972d18bc523679b into 96d982b727b14be09c820f476e2bf978(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:36,467 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:36,467 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., storeName=f4469ee439a16ac91972d18bc523679b/C, priority=13, startTime=1733777016007; duration=0sec 2024-12-09T20:43:36,467 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:36,467 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4469ee439a16ac91972d18bc523679b:C 2024-12-09T20:43:36,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2ea0865206014ff790aad8a423a32b21 2024-12-09T20:43:36,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/43e1f843a2284d478a6b74e87c9bd577 is 50, key is test_row_0/C:col10/1733777015509/Put/seqid=0 2024-12-09T20:43:36,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742246_1422 (size=12301) 2024-12-09T20:43:36,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59588 deadline: 1733777076671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59628 deadline: 1733777076671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59642 deadline: 1733777076671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59612 deadline: 1733777076701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:36,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:36,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59604 deadline: 1733777076751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:43:36,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:36,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:36,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/43e1f843a2284d478a6b74e87c9bd577 2024-12-09T20:43:36,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/47b1af39a9c3499aa068196059b81e8b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/47b1af39a9c3499aa068196059b81e8b 2024-12-09T20:43:36,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/47b1af39a9c3499aa068196059b81e8b, entries=150, sequenceid=607, filesize=30.5 K 2024-12-09T20:43:36,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/2ea0865206014ff790aad8a423a32b21 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2ea0865206014ff790aad8a423a32b21 2024-12-09T20:43:36,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2ea0865206014ff790aad8a423a32b21, entries=150, sequenceid=607, filesize=12.0 K 2024-12-09T20:43:36,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/43e1f843a2284d478a6b74e87c9bd577 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/43e1f843a2284d478a6b74e87c9bd577 2024-12-09T20:43:36,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/43e1f843a2284d478a6b74e87c9bd577, entries=150, sequenceid=607, filesize=12.0 K 2024-12-09T20:43:36,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~449.67 KB/460464, currentSize=26.84 KB/27480 for f4469ee439a16ac91972d18bc523679b in 864ms, sequenceid=607, compaction requested=false 2024-12-09T20:43:36,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:37,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T20:43:37,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:37,049 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T20:43:37,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:37,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:37,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:37,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:37,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:37,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:37,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209794450a331d949a5a3db34fa6cbaf7d6_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_1/A:col10/1733777016105/Put/seqid=0 2024-12-09T20:43:37,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742247_1423 (size=9914) 2024-12-09T20:43:37,266 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-12-09T20:43:37,266 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:37,266 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,266 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:37,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:37,477 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209794450a331d949a5a3db34fa6cbaf7d6_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209794450a331d949a5a3db34fa6cbaf7d6_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:37,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/15acf58f95ab45d9afc28198ee1d6000, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/15acf58f95ab45d9afc28198ee1d6000 is 175, key is test_row_1/A:col10/1733777016105/Put/seqid=0 2024-12-09T20:43:37,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742248_1424 (size=22561) 2024-12-09T20:43:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:37,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. as already flushing 2024-12-09T20:43:37,684 INFO [Thread-1529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1529. 2024-12-09T20:43:37,684 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-12-09T20:43:37,684 DEBUG [Thread-1529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:37,684 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:37,684 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,684 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,684 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,684 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:37,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:37,693 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-12-09T20:43:37,693 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:37,693 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,694 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:37,715 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-12-09T20:43:37,715 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:43:37,715 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,715 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:37,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:37,884 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=616, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/15acf58f95ab45d9afc28198ee1d6000 2024-12-09T20:43:37,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/508a76ebefae4d269445f0c51fc787d6 is 50, key is test_row_1/B:col10/1733777016105/Put/seqid=0 2024-12-09T20:43:37,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742249_1425 (size=9857) 2024-12-09T20:43:37,961 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/c55432042af94d00922ec36d33b56fd8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e26a01ecd0a741d18fc8c3a827ff6d2b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f93a70f707a849558b51be9a3e5ea4cf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/2de4d5a6b86a401fbf398f1cebde8db8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/abbf767ad55a4be28163dcc56d0bb3c0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/37f8c0fbd2254987ad694b4baccec24d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/41549d18af2e40d8ab67740fd37f7f23, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/739097874fec422aa33fa861fb9609a9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/5e493ee58acf420e9626b2ee8d644cfc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e1f99a8cd65e40e0b0e9f1e5ccf40f6a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e14b3f8df46e421e9fa644e11d10e5e3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/96495040821e43f0b6d9d22321b8d9cb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/926caf710f2e492983a8f2275637f5d4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/18217abb5af84797b67ad4da2c71d2b0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/1191a6ca786b4d4bb08434b038e1731c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/360d4f2068a147118ac83bfe0bf3f123, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d050b29c826c4758a20d47dc04301c66, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/bdd1e02292b748d4b29515cc035af9df, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/982afab06f3949d6ade2699dc806e0a5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/fa149032c7b941d48879c27671c680b1] to archive 2024-12-09T20:43:37,962 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:43:37,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/c55432042af94d00922ec36d33b56fd8 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/c55432042af94d00922ec36d33b56fd8 2024-12-09T20:43:37,963 WARN [regionserver/00c23a7251f8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-09T20:43:37,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e26a01ecd0a741d18fc8c3a827ff6d2b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e26a01ecd0a741d18fc8c3a827ff6d2b 2024-12-09T20:43:37,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f93a70f707a849558b51be9a3e5ea4cf to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f93a70f707a849558b51be9a3e5ea4cf 2024-12-09T20:43:37,966 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/2de4d5a6b86a401fbf398f1cebde8db8 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/2de4d5a6b86a401fbf398f1cebde8db8 2024-12-09T20:43:37,966 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/abbf767ad55a4be28163dcc56d0bb3c0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/abbf767ad55a4be28163dcc56d0bb3c0 2024-12-09T20:43:37,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/37f8c0fbd2254987ad694b4baccec24d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/37f8c0fbd2254987ad694b4baccec24d 2024-12-09T20:43:37,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/41549d18af2e40d8ab67740fd37f7f23 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/41549d18af2e40d8ab67740fd37f7f23 2024-12-09T20:43:37,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/739097874fec422aa33fa861fb9609a9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/739097874fec422aa33fa861fb9609a9 2024-12-09T20:43:37,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/5e493ee58acf420e9626b2ee8d644cfc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/5e493ee58acf420e9626b2ee8d644cfc 2024-12-09T20:43:37,970 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e1f99a8cd65e40e0b0e9f1e5ccf40f6a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e1f99a8cd65e40e0b0e9f1e5ccf40f6a 2024-12-09T20:43:37,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e14b3f8df46e421e9fa644e11d10e5e3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/e14b3f8df46e421e9fa644e11d10e5e3 2024-12-09T20:43:37,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/96495040821e43f0b6d9d22321b8d9cb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/96495040821e43f0b6d9d22321b8d9cb 2024-12-09T20:43:37,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/926caf710f2e492983a8f2275637f5d4 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/926caf710f2e492983a8f2275637f5d4 2024-12-09T20:43:37,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/18217abb5af84797b67ad4da2c71d2b0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/18217abb5af84797b67ad4da2c71d2b0 2024-12-09T20:43:37,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/1191a6ca786b4d4bb08434b038e1731c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/1191a6ca786b4d4bb08434b038e1731c 2024-12-09T20:43:37,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/360d4f2068a147118ac83bfe0bf3f123 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/360d4f2068a147118ac83bfe0bf3f123 2024-12-09T20:43:37,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d050b29c826c4758a20d47dc04301c66 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d050b29c826c4758a20d47dc04301c66 2024-12-09T20:43:37,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/bdd1e02292b748d4b29515cc035af9df to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/bdd1e02292b748d4b29515cc035af9df 2024-12-09T20:43:37,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/982afab06f3949d6ade2699dc806e0a5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/982afab06f3949d6ade2699dc806e0a5 2024-12-09T20:43:37,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/fa149032c7b941d48879c27671c680b1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/fa149032c7b941d48879c27671c680b1 2024-12-09T20:43:37,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/a57eeee09d604d0a8163cb2fca223efa, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/f566dd219c4846b79fb822685af6ba05, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/99fb21457d75434d9a310c77bcc85a16, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/910674ab28ea46ffb3805b426880fc97, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8d4f0930d38e4ca2ac9c555cf7b543d2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/afdefbab1ee4447a89739101579dd825, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2bc71931f4004467b74aa60cd4d1344a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/679a45832668454d854dcf1a92ff2217, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2b6c9c29633c435c8a738258734703d1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bdcc8c3188f541fd97f902f5da670109, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/1526fa63f8cb42f58cbcca722f78d64b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/05174a42df614d278558a307ba97ca32, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/d5f3ac4388834ee5b7367b8d23e0d8bc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8b83d84d4b4f4e6ebb39621a5f7ac598, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bae08ce230fc4c64b920b73168b5a682, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/123606b5b78744b59296396ff014b1f7, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/42c4a35677d64d11b7fe80d8c69970d0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/3ea9a6c6f6c941c5a294b458f7daa84c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8c6475aeffbe412f8e3b64c8ba5c442e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2903fd8134004951831b96b89b90cdec] to archive 2024-12-09T20:43:37,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:43:37,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/a57eeee09d604d0a8163cb2fca223efa to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/a57eeee09d604d0a8163cb2fca223efa 2024-12-09T20:43:37,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/f566dd219c4846b79fb822685af6ba05 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/f566dd219c4846b79fb822685af6ba05 2024-12-09T20:43:37,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/99fb21457d75434d9a310c77bcc85a16 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/99fb21457d75434d9a310c77bcc85a16 2024-12-09T20:43:37,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/910674ab28ea46ffb3805b426880fc97 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/910674ab28ea46ffb3805b426880fc97 2024-12-09T20:43:37,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8d4f0930d38e4ca2ac9c555cf7b543d2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8d4f0930d38e4ca2ac9c555cf7b543d2 2024-12-09T20:43:37,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/afdefbab1ee4447a89739101579dd825 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/afdefbab1ee4447a89739101579dd825 2024-12-09T20:43:37,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2bc71931f4004467b74aa60cd4d1344a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2bc71931f4004467b74aa60cd4d1344a 2024-12-09T20:43:37,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/679a45832668454d854dcf1a92ff2217 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/679a45832668454d854dcf1a92ff2217 2024-12-09T20:43:37,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2b6c9c29633c435c8a738258734703d1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2b6c9c29633c435c8a738258734703d1 2024-12-09T20:43:37,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bdcc8c3188f541fd97f902f5da670109 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bdcc8c3188f541fd97f902f5da670109 2024-12-09T20:43:37,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/1526fa63f8cb42f58cbcca722f78d64b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/1526fa63f8cb42f58cbcca722f78d64b 2024-12-09T20:43:37,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/05174a42df614d278558a307ba97ca32 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/05174a42df614d278558a307ba97ca32 2024-12-09T20:43:37,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/d5f3ac4388834ee5b7367b8d23e0d8bc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/d5f3ac4388834ee5b7367b8d23e0d8bc 2024-12-09T20:43:38,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8b83d84d4b4f4e6ebb39621a5f7ac598 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8b83d84d4b4f4e6ebb39621a5f7ac598 2024-12-09T20:43:38,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bae08ce230fc4c64b920b73168b5a682 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/bae08ce230fc4c64b920b73168b5a682 2024-12-09T20:43:38,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/123606b5b78744b59296396ff014b1f7 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/123606b5b78744b59296396ff014b1f7 2024-12-09T20:43:38,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/42c4a35677d64d11b7fe80d8c69970d0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/42c4a35677d64d11b7fe80d8c69970d0 2024-12-09T20:43:38,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/3ea9a6c6f6c941c5a294b458f7daa84c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/3ea9a6c6f6c941c5a294b458f7daa84c 2024-12-09T20:43:38,006 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8c6475aeffbe412f8e3b64c8ba5c442e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/8c6475aeffbe412f8e3b64c8ba5c442e 2024-12-09T20:43:38,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2903fd8134004951831b96b89b90cdec to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2903fd8134004951831b96b89b90cdec 2024-12-09T20:43:38,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6a99e18bf4734257bfc6028097f1606f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4c0d624b6e1e48528dc6eb32f000c0f6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/57f01127f71e40349cf3dcaa97a52cf9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5d9ed05c658a4575b630c019c5ac8f9e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/3ca4f2d1867e400eaaa23b5e801b55dd, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/83d46b6d7ab842ceb15f5b95a704dfef, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d7d4df8962254269a000392fa3a595c6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4653d563d86645b483b0b9ff579935b5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/cf3b297c90034764b7eb0c92624b20ff, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5acc1c80f74341fa856068bd15da14f1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e21bfb4439334144a7cdeae0bce7546f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e23d96266c8d4b4e969ead3d757c3491, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d06cd967e5c947d994abde0aa5a2de82, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6746aadbb0fb48d1b21017b35b0d8833, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/fb480993c89d4688803dc4eaf64114c4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/1265e8dbfcea47e0b39d0efcc7490e61, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/59f254cc883c472c80073b599c3e8367, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/b87983569a754736a9eb97b594d4e4af, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/9ee8a01bc66a48338d3104084815793b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/71afb51eb0ab439381d106ecc852eec9] to archive 2024-12-09T20:43:38,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:43:38,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6a99e18bf4734257bfc6028097f1606f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6a99e18bf4734257bfc6028097f1606f 2024-12-09T20:43:38,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4c0d624b6e1e48528dc6eb32f000c0f6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4c0d624b6e1e48528dc6eb32f000c0f6 2024-12-09T20:43:38,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/57f01127f71e40349cf3dcaa97a52cf9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/57f01127f71e40349cf3dcaa97a52cf9 2024-12-09T20:43:38,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5d9ed05c658a4575b630c019c5ac8f9e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5d9ed05c658a4575b630c019c5ac8f9e 2024-12-09T20:43:38,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/3ca4f2d1867e400eaaa23b5e801b55dd to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/3ca4f2d1867e400eaaa23b5e801b55dd 2024-12-09T20:43:38,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/83d46b6d7ab842ceb15f5b95a704dfef to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/83d46b6d7ab842ceb15f5b95a704dfef 2024-12-09T20:43:38,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d7d4df8962254269a000392fa3a595c6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d7d4df8962254269a000392fa3a595c6 2024-12-09T20:43:38,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4653d563d86645b483b0b9ff579935b5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/4653d563d86645b483b0b9ff579935b5 2024-12-09T20:43:38,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/cf3b297c90034764b7eb0c92624b20ff to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/cf3b297c90034764b7eb0c92624b20ff 2024-12-09T20:43:38,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5acc1c80f74341fa856068bd15da14f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/5acc1c80f74341fa856068bd15da14f1 2024-12-09T20:43:38,022 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e21bfb4439334144a7cdeae0bce7546f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e21bfb4439334144a7cdeae0bce7546f 2024-12-09T20:43:38,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e23d96266c8d4b4e969ead3d757c3491 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/e23d96266c8d4b4e969ead3d757c3491 2024-12-09T20:43:38,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d06cd967e5c947d994abde0aa5a2de82 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/d06cd967e5c947d994abde0aa5a2de82 2024-12-09T20:43:38,024 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6746aadbb0fb48d1b21017b35b0d8833 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6746aadbb0fb48d1b21017b35b0d8833 2024-12-09T20:43:38,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/fb480993c89d4688803dc4eaf64114c4 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/fb480993c89d4688803dc4eaf64114c4 2024-12-09T20:43:38,027 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/1265e8dbfcea47e0b39d0efcc7490e61 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/1265e8dbfcea47e0b39d0efcc7490e61 2024-12-09T20:43:38,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/59f254cc883c472c80073b599c3e8367 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/59f254cc883c472c80073b599c3e8367 2024-12-09T20:43:38,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/b87983569a754736a9eb97b594d4e4af to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/b87983569a754736a9eb97b594d4e4af 2024-12-09T20:43:38,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/9ee8a01bc66a48338d3104084815793b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/9ee8a01bc66a48338d3104084815793b 2024-12-09T20:43:38,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/00c23a7251f8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/71afb51eb0ab439381d106ecc852eec9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/71afb51eb0ab439381d106ecc852eec9 2024-12-09T20:43:38,301 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/508a76ebefae4d269445f0c51fc787d6 2024-12-09T20:43:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/6196ff4536fa46c397d72993b3f7cbe1 is 50, key is test_row_1/C:col10/1733777016105/Put/seqid=0 2024-12-09T20:43:38,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742250_1426 (size=9857) 2024-12-09T20:43:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T20:43:38,714 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/6196ff4536fa46c397d72993b3f7cbe1 2024-12-09T20:43:38,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/15acf58f95ab45d9afc28198ee1d6000 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/15acf58f95ab45d9afc28198ee1d6000 2024-12-09T20:43:38,721 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/15acf58f95ab45d9afc28198ee1d6000, entries=100, sequenceid=616, filesize=22.0 K 2024-12-09T20:43:38,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/508a76ebefae4d269445f0c51fc787d6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/508a76ebefae4d269445f0c51fc787d6 2024-12-09T20:43:38,724 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/508a76ebefae4d269445f0c51fc787d6, entries=100, sequenceid=616, filesize=9.6 K 2024-12-09T20:43:38,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/6196ff4536fa46c397d72993b3f7cbe1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6196ff4536fa46c397d72993b3f7cbe1 2024-12-09T20:43:38,727 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6196ff4536fa46c397d72993b3f7cbe1, entries=100, sequenceid=616, filesize=9.6 K 2024-12-09T20:43:38,728 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=33.54 KB/34350 for f4469ee439a16ac91972d18bc523679b in 1679ms, sequenceid=616, compaction requested=true 2024-12-09T20:43:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for f4469ee439a16ac91972d18bc523679b: 2024-12-09T20:43:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-09T20:43:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-09T20:43:38,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-09T20:43:38,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4420 sec 2024-12-09T20:43:38,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 4.4480 sec 2024-12-09T20:43:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T20:43:42,462 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 261 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1364 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4092 rows 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1372 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4074 rows 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1353 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4059 rows 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1384 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4068 rows 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1393 2024-12-09T20:43:42,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4108 rows 2024-12-09T20:43:42,463 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:43:42,464 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:43:42,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:42,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:42,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:43:42,466 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T20:43:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T20:43:42,472 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777022472"}]},"ts":"1733777022472"} 2024-12-09T20:43:42,474 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T20:43:42,474 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T20:43:42,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T20:43:42,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, UNASSIGN}] 2024-12-09T20:43:42,477 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, UNASSIGN 2024-12-09T20:43:42,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=f4469ee439a16ac91972d18bc523679b, regionState=CLOSING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:42,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, UNASSIGN because future has completed 2024-12-09T20:43:42,480 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T20:43:42,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:43:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T20:43:42,633 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:42,633 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T20:43:42,633 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing f4469ee439a16ac91972d18bc523679b, disabling compactions & flushes 2024-12-09T20:43:42,633 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:42,633 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:42,633 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. after waiting 0 ms 2024-12-09T20:43:42,633 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:42,633 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing f4469ee439a16ac91972d18bc523679b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T20:43:42,634 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=A 2024-12-09T20:43:42,634 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:42,634 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=B 2024-12-09T20:43:42,634 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:42,634 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4469ee439a16ac91972d18bc523679b, store=C 2024-12-09T20:43:42,634 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:42,642 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b48c186cbad141ce9e121974c8af9364_f4469ee439a16ac91972d18bc523679b is 50, key is test_row_0/A:col10/1733777017692/Put/seqid=0 2024-12-09T20:43:42,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742251_1427 (size=12454) 2024-12-09T20:43:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T20:43:43,048 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:43,052 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b48c186cbad141ce9e121974c8af9364_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b48c186cbad141ce9e121974c8af9364_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:43,052 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/d45563d123e14a1980814aaa7fcc0313, store: [table=TestAcidGuarantees family=A region=f4469ee439a16ac91972d18bc523679b] 2024-12-09T20:43:43,053 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/d45563d123e14a1980814aaa7fcc0313 is 175, key is test_row_0/A:col10/1733777017692/Put/seqid=0 2024-12-09T20:43:43,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742252_1428 (size=31255) 2024-12-09T20:43:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T20:43:43,457 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=624, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/d45563d123e14a1980814aaa7fcc0313 2024-12-09T20:43:43,463 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/7706333d26a54d7fb63fd0efc85baeab is 50, key is test_row_0/B:col10/1733777017692/Put/seqid=0 2024-12-09T20:43:43,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742253_1429 (size=12301) 2024-12-09T20:43:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T20:43:43,867 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/7706333d26a54d7fb63fd0efc85baeab 2024-12-09T20:43:43,878 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/586586d8d86c433291f19c89ecb9b4c5 is 50, key is test_row_0/C:col10/1733777017692/Put/seqid=0 2024-12-09T20:43:43,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742254_1430 (size=12301) 2024-12-09T20:43:44,283 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/586586d8d86c433291f19c89ecb9b4c5 2024-12-09T20:43:44,289 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/A/d45563d123e14a1980814aaa7fcc0313 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d45563d123e14a1980814aaa7fcc0313 2024-12-09T20:43:44,294 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d45563d123e14a1980814aaa7fcc0313, entries=150, sequenceid=624, filesize=30.5 K 2024-12-09T20:43:44,294 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/B/7706333d26a54d7fb63fd0efc85baeab as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/7706333d26a54d7fb63fd0efc85baeab 2024-12-09T20:43:44,297 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/7706333d26a54d7fb63fd0efc85baeab, entries=150, sequenceid=624, filesize=12.0 K 2024-12-09T20:43:44,298 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/.tmp/C/586586d8d86c433291f19c89ecb9b4c5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/586586d8d86c433291f19c89ecb9b4c5 2024-12-09T20:43:44,301 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/586586d8d86c433291f19c89ecb9b4c5, entries=150, sequenceid=624, filesize=12.0 K 2024-12-09T20:43:44,301 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f4469ee439a16ac91972d18bc523679b in 1668ms, sequenceid=624, compaction requested=true 2024-12-09T20:43:44,307 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/recovered.edits/627.seqid, newMaxSeqId=627, maxSeqId=4 2024-12-09T20:43:44,308 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. 2024-12-09T20:43:44,308 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for f4469ee439a16ac91972d18bc523679b: Waiting for close lock at 1733777022633Running coprocessor pre-close hooks at 1733777022633Disabling compacts and flushes for region at 1733777022633Disabling writes for close at 1733777022633Obtaining lock to block concurrent updates at 1733777022633Preparing flush snapshotting stores in f4469ee439a16ac91972d18bc523679b at 1733777022633Finished memstore snapshotting TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733777022634 (+1 ms)Flushing stores of TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b. at 1733777022634Flushing f4469ee439a16ac91972d18bc523679b/A: creating writer at 1733777022635 (+1 ms)Flushing mob file f4469ee439a16ac91972d18bc523679b/A: appending metadata at 1733777022642 (+7 ms)Flushing mob file f4469ee439a16ac91972d18bc523679b/A: closing flushed file at 1733777022642Flushing f4469ee439a16ac91972d18bc523679b/A: appending metadata at 1733777023053 (+411 ms)Flushing f4469ee439a16ac91972d18bc523679b/A: closing flushed file at 1733777023053Flushing f4469ee439a16ac91972d18bc523679b/B: creating writer at 1733777023460 (+407 ms)Flushing f4469ee439a16ac91972d18bc523679b/B: appending metadata at 1733777023462 (+2 ms)Flushing f4469ee439a16ac91972d18bc523679b/B: closing flushed file at 1733777023463 (+1 ms)Flushing f4469ee439a16ac91972d18bc523679b/C: creating writer at 1733777023874 (+411 ms)Flushing f4469ee439a16ac91972d18bc523679b/C: appending metadata at 1733777023878 (+4 ms)Flushing f4469ee439a16ac91972d18bc523679b/C: closing flushed file at 1733777023878Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b88714: reopening flushed file at 1733777024288 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38fcaa1e: reopening flushed file at 1733777024294 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@968ba8c: reopening flushed file at 1733777024297 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f4469ee439a16ac91972d18bc523679b in 1668ms, sequenceid=624, compaction requested=true at 1733777024301 (+4 ms)Writing region close event to WAL at 1733777024305 (+4 ms)Running coprocessor post-close hooks at 1733777024308 (+3 ms)Closed at 1733777024308 2024-12-09T20:43:44,310 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,310 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=f4469ee439a16ac91972d18bc523679b, regionState=CLOSED 2024-12-09T20:43:44,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:43:44,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-09T20:43:44,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure f4469ee439a16ac91972d18bc523679b, server=00c23a7251f8,36519,1733776895755 in 1.8320 sec 2024-12-09T20:43:44,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-09T20:43:44,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4469ee439a16ac91972d18bc523679b, UNASSIGN in 1.8370 sec 2024-12-09T20:43:44,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-09T20:43:44,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8410 sec 2024-12-09T20:43:44,318 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777024318"}]},"ts":"1733777024318"} 2024-12-09T20:43:44,319 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T20:43:44,320 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T20:43:44,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8540 sec 2024-12-09T20:43:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T20:43:44,622 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:44,623 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T20:43:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:44,627 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T20:43:44,629 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:44,634 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,638 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/recovered.edits] 2024-12-09T20:43:44,642 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/15acf58f95ab45d9afc28198ee1d6000 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/15acf58f95ab45d9afc28198ee1d6000 2024-12-09T20:43:44,643 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/47b1af39a9c3499aa068196059b81e8b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/47b1af39a9c3499aa068196059b81e8b 2024-12-09T20:43:44,644 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d45563d123e14a1980814aaa7fcc0313 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/d45563d123e14a1980814aaa7fcc0313 2024-12-09T20:43:44,645 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f3787792b6834c2491fbae2779f038c6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/A/f3787792b6834c2491fbae2779f038c6 2024-12-09T20:43:44,646 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2ea0865206014ff790aad8a423a32b21 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/2ea0865206014ff790aad8a423a32b21 2024-12-09T20:43:44,647 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/508a76ebefae4d269445f0c51fc787d6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/508a76ebefae4d269445f0c51fc787d6 2024-12-09T20:43:44,648 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/5b601f5eb2b54277b3622d6e6186fe11 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/5b601f5eb2b54277b3622d6e6186fe11 2024-12-09T20:43:44,649 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/7706333d26a54d7fb63fd0efc85baeab to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/B/7706333d26a54d7fb63fd0efc85baeab 2024-12-09T20:43:44,650 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/43e1f843a2284d478a6b74e87c9bd577 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/43e1f843a2284d478a6b74e87c9bd577 2024-12-09T20:43:44,651 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/586586d8d86c433291f19c89ecb9b4c5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/586586d8d86c433291f19c89ecb9b4c5 2024-12-09T20:43:44,652 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6196ff4536fa46c397d72993b3f7cbe1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/6196ff4536fa46c397d72993b3f7cbe1 2024-12-09T20:43:44,652 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/96d982b727b14be09c820f476e2bf978 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/C/96d982b727b14be09c820f476e2bf978 2024-12-09T20:43:44,654 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/recovered.edits/627.seqid to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b/recovered.edits/627.seqid 2024-12-09T20:43:44,654 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,655 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T20:43:44,655 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T20:43:44,655 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T20:43:44,657 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090b0efc5254fe4f1ea616410e9e872706_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090b0efc5254fe4f1ea616410e9e872706_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,658 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120919025525e0e347c3b3e13597f4c25d0c_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120919025525e0e347c3b3e13597f4c25d0c_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,659 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091a58480939f04fac9f580049064e1cac_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091a58480939f04fac9f580049064e1cac_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,660 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209299db223fb134458a4fd0037b459b9c8_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209299db223fb134458a4fd0037b459b9c8_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,661 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120952abc6e2f7264fe7ae3919867f8271a7_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120952abc6e2f7264fe7ae3919867f8271a7_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,661 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120955330a9e74e14aee829cad4d5be66936_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120955330a9e74e14aee829cad4d5be66936_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,662 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120956e41c9ea0c34e25a9a6f5f3dd3c6093_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120956e41c9ea0c34e25a9a6f5f3dd3c6093_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,663 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120960dfc2dfa75c45838da2c49b4684ab8b_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120960dfc2dfa75c45838da2c49b4684ab8b_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,664 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096c933eec47214de89b8aeaa01b80f901_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096c933eec47214de89b8aeaa01b80f901_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,664 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097518805bd8474d6f80ab9f9c1d715f61_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097518805bd8474d6f80ab9f9c1d715f61_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,665 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209774057ff5bfd449baeb17b6469817825_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209774057ff5bfd449baeb17b6469817825_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,666 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209794450a331d949a5a3db34fa6cbaf7d6_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209794450a331d949a5a3db34fa6cbaf7d6_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,666 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098383bf7090d3470ba664114f37092f0a_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098383bf7090d3470ba664114f37092f0a_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,667 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209889b29af17084864ae0d77b9b52bb312_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209889b29af17084864ae0d77b9b52bb312_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,668 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a2409c679181498d93d9665be8d700e7_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a2409c679181498d93d9665be8d700e7_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,668 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a58e3d2e4f6c484b83bf28422a298ab6_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a58e3d2e4f6c484b83bf28422a298ab6_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,669 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b48c186cbad141ce9e121974c8af9364_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b48c186cbad141ce9e121974c8af9364_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,670 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ce745826655c4cc7bc825d3d86ea9f2d_f4469ee439a16ac91972d18bc523679b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ce745826655c4cc7bc825d3d86ea9f2d_f4469ee439a16ac91972d18bc523679b 2024-12-09T20:43:44,670 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T20:43:44,672 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:44,673 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T20:43:44,674 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T20:43:44,675 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:44,675 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T20:43:44,676 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733777024675"}]},"ts":"9223372036854775807"} 2024-12-09T20:43:44,677 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T20:43:44,677 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f4469ee439a16ac91972d18bc523679b, NAME => 'TestAcidGuarantees,,1733776992958.f4469ee439a16ac91972d18bc523679b.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T20:43:44,677 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T20:43:44,677 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733777024677"}]},"ts":"9223372036854775807"} 2024-12-09T20:43:44,678 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T20:43:44,679 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:44,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-12-09T20:43:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T20:43:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T20:43:44,732 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:44,740 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 587) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2776 (was 2892) 2024-12-09T20:43:44,748 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2775 2024-12-09T20:43:44,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T20:43:44,749 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:43:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T20:43:44,750 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:43:44,751 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:44,751 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-12-09T20:43:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T20:43:44,752 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:43:44,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742255_1431 (size=1061) 2024-12-09T20:43:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T20:43:45,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T20:43:45,158 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948 2024-12-09T20:43:45,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742256_1432 (size=53) 2024-12-09T20:43:45,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T20:43:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T20:43:45,562 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:43:45,562 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 879ac9ed3549bd954296156f21c08ad4, disabling compactions & flushes 2024-12-09T20:43:45,562 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:45,562 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:45,562 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. after waiting 0 ms 2024-12-09T20:43:45,562 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:45,562 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:45,562 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 879ac9ed3549bd954296156f21c08ad4: Waiting for close lock at 1733777025562Disabling compacts and flushes for region at 1733777025562Disabling writes for close at 1733777025562Writing region close event to WAL at 1733777025562Closed at 1733777025562 2024-12-09T20:43:45,563 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:43:45,564 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733777025564"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733777025564"}]},"ts":"1733777025564"} 2024-12-09T20:43:45,565 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:43:45,566 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:43:45,567 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777025566"}]},"ts":"1733777025566"} 2024-12-09T20:43:45,568 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T20:43:45,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=879ac9ed3549bd954296156f21c08ad4, ASSIGN}] 2024-12-09T20:43:45,570 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=879ac9ed3549bd954296156f21c08ad4, ASSIGN 2024-12-09T20:43:45,570 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=879ac9ed3549bd954296156f21c08ad4, ASSIGN; state=OFFLINE, location=00c23a7251f8,36519,1733776895755; forceNewPlan=false, retain=false 2024-12-09T20:43:45,721 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=879ac9ed3549bd954296156f21c08ad4, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:45,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=879ac9ed3549bd954296156f21c08ad4, ASSIGN because future has completed 2024-12-09T20:43:45,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:43:45,878 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:45,878 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:43:45,878 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,878 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:43:45,879 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,879 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,880 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,881 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:43:45,881 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 879ac9ed3549bd954296156f21c08ad4 columnFamilyName A 2024-12-09T20:43:45,881 DEBUG [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:45,882 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] regionserver.HStore(327): Store=879ac9ed3549bd954296156f21c08ad4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:43:45,882 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,883 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:43:45,883 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 879ac9ed3549bd954296156f21c08ad4 columnFamilyName B 2024-12-09T20:43:45,883 DEBUG [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:45,883 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] regionserver.HStore(327): Store=879ac9ed3549bd954296156f21c08ad4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:43:45,884 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,884 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:43:45,885 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 879ac9ed3549bd954296156f21c08ad4 columnFamilyName C 2024-12-09T20:43:45,885 DEBUG [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:43:45,885 INFO [StoreOpener-879ac9ed3549bd954296156f21c08ad4-1 {}] regionserver.HStore(327): Store=879ac9ed3549bd954296156f21c08ad4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:43:45,885 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:45,885 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,886 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,886 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,887 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,887 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,888 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:43:45,889 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,891 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:43:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T20:43:45,891 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 879ac9ed3549bd954296156f21c08ad4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74614950, jitterRate=0.11184939742088318}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:43:45,891 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:45,892 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 879ac9ed3549bd954296156f21c08ad4: Running coprocessor pre-open hook at 1733777025879Writing region info on filesystem at 1733777025879Initializing all the Stores at 1733777025879Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733777025879Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733777025880 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733777025880Cleaning up temporary data from old regions at 1733777025887 (+7 ms)Running coprocessor post-open hooks at 1733777025891 (+4 ms)Region opened successfully at 1733777025892 (+1 ms) 2024-12-09T20:43:45,893 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., pid=120, masterSystemTime=1733777025875 2024-12-09T20:43:45,894 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:45,894 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:45,895 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=879ac9ed3549bd954296156f21c08ad4, regionState=OPEN, openSeqNum=2, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:45,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:43:45,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-09T20:43:45,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 in 174 msec 2024-12-09T20:43:45,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-12-09T20:43:45,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=879ac9ed3549bd954296156f21c08ad4, ASSIGN in 330 msec 2024-12-09T20:43:45,901 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:43:45,901 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777025901"}]},"ts":"1733777025901"} 2024-12-09T20:43:45,903 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T20:43:45,904 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:43:45,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-12-09T20:43:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T20:43:46,902 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:46,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,904 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@584ecb58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,905 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7682d7d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,906 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,908 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T20:43:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c23a7251f8,42351,1733776895022 2024-12-09T20:43:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14dc5333 2024-12-09T20:43:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T20:43:46,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T20:43:46,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d965e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42cc479e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a7ab96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747190bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25357048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3b43ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,921 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c48468, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,922 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a26fea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,924 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c3bfba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,925 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a94d4e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,927 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d583fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,929 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@efca916, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,931 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32dfdce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,933 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26475238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,935 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d474ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,937 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62508fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,940 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d41ad2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,941 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4426ed6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:43:46,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:43:46,944 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:43:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:43:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:43:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@502b70d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:43:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:43:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:43:46,945 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:43:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400701ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:43:46,946 DEBUG [Thread-1959 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,946 DEBUG [Thread-1948 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,946 DEBUG [Thread-1963 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,946 DEBUG [Thread-1950 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,947 DEBUG [Thread-1952 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,947 DEBUG [Thread-1946 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,947 DEBUG [Thread-1957 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,948 DEBUG [Thread-1965 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,948 DEBUG [Thread-1954 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,949 DEBUG [Thread-1954 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,949 DEBUG [Thread-1954 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,952 DEBUG [Thread-1950 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,952 DEBUG [Thread-1950 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-09T20:43:46,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,955 DEBUG [Thread-1950 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T20:43:46,956 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,957 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:46,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:46,958 DEBUG [Thread-1961 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:43:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:43:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:43:46,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:43:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2] 2024-12-09T20:43:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:46,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T20:43:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:46,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/e598b860d9054457b86f3bfbf2ad98ff is 50, key is test_row_0/A:col10/1733777026965/Put/seqid=0 2024-12-09T20:43:46,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:46,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777086988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:46,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742257_1433 (size=14341) 2024-12-09T20:43:47,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777086989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777086990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777086991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777086993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T20:43:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777087101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T20:43:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777087111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777087111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777087112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777087112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T20:43:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T20:43:47,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777087311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777087322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777087323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777087323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777087331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/e598b860d9054457b86f3bfbf2ad98ff 2024-12-09T20:43:47,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T20:43:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/20f7e61568ec4ce1a6ee8bbc61bc3e23 is 50, key is test_row_0/B:col10/1733777026965/Put/seqid=0 2024-12-09T20:43:47,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742258_1434 (size=12001) 2024-12-09T20:43:47,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T20:43:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T20:43:47,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777087621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777087631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777087631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777087641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777087651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:47,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T20:43:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/20f7e61568ec4ce1a6ee8bbc61bc3e23 2024-12-09T20:43:47,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/7b8e047cab444a68ad7605eb4b28a644 is 50, key is test_row_0/C:col10/1733777026965/Put/seqid=0 2024-12-09T20:43:47,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742259_1435 (size=12001) 2024-12-09T20:43:47,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T20:43:47,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:47,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:47,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:48,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T20:43:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:48,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:48,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T20:43:48,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777088131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:48,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777088141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777088141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:48,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777088151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:48,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777088171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:48,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T20:43:48,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:48,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:48,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:48,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:48,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:48,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/7b8e047cab444a68ad7605eb4b28a644 2024-12-09T20:43:48,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/e598b860d9054457b86f3bfbf2ad98ff as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/e598b860d9054457b86f3bfbf2ad98ff 2024-12-09T20:43:48,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/e598b860d9054457b86f3bfbf2ad98ff, entries=200, sequenceid=13, filesize=14.0 K 2024-12-09T20:43:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/20f7e61568ec4ce1a6ee8bbc61bc3e23 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/20f7e61568ec4ce1a6ee8bbc61bc3e23 2024-12-09T20:43:48,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/20f7e61568ec4ce1a6ee8bbc61bc3e23, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T20:43:48,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/7b8e047cab444a68ad7605eb4b28a644 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7b8e047cab444a68ad7605eb4b28a644 2024-12-09T20:43:48,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7b8e047cab444a68ad7605eb4b28a644, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T20:43:48,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 879ac9ed3549bd954296156f21c08ad4 in 1317ms, sequenceid=13, compaction requested=false 2024-12-09T20:43:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:48,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T20:43:48,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:48,342 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:43:48,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:48,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:48,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:48,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:48,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:48,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:48,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/7ac343aa9389497fba2d527f6b39b717 is 50, key is test_row_0/A:col10/1733777026990/Put/seqid=0 2024-12-09T20:43:48,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742260_1436 (size=12001) 2024-12-09T20:43:48,757 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/7ac343aa9389497fba2d527f6b39b717 2024-12-09T20:43:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/23278f6d89ff483d95957f8ad290e370 is 50, key is test_row_0/B:col10/1733777026990/Put/seqid=0 2024-12-09T20:43:48,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742261_1437 (size=12001) 2024-12-09T20:43:48,803 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/23278f6d89ff483d95957f8ad290e370 2024-12-09T20:43:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/eb054c3dce9e42a195de1ce4357ee3e4 is 50, key is test_row_0/C:col10/1733777026990/Put/seqid=0 2024-12-09T20:43:48,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742262_1438 (size=12001) 2024-12-09T20:43:48,853 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/eb054c3dce9e42a195de1ce4357ee3e4 2024-12-09T20:43:48,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/7ac343aa9389497fba2d527f6b39b717 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/7ac343aa9389497fba2d527f6b39b717 2024-12-09T20:43:48,863 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/7ac343aa9389497fba2d527f6b39b717, entries=150, sequenceid=33, filesize=11.7 K 2024-12-09T20:43:48,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/23278f6d89ff483d95957f8ad290e370 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/23278f6d89ff483d95957f8ad290e370 2024-12-09T20:43:48,868 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/23278f6d89ff483d95957f8ad290e370, entries=150, sequenceid=33, filesize=11.7 K 2024-12-09T20:43:48,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/eb054c3dce9e42a195de1ce4357ee3e4 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/eb054c3dce9e42a195de1ce4357ee3e4 2024-12-09T20:43:48,873 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/eb054c3dce9e42a195de1ce4357ee3e4, entries=150, sequenceid=33, filesize=11.7 K 2024-12-09T20:43:48,876 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 879ac9ed3549bd954296156f21c08ad4 in 534ms, sequenceid=33, compaction requested=false 2024-12-09T20:43:48,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:48,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:48,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-09T20:43:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-09T20:43:48,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-09T20:43:48,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9200 sec 2024-12-09T20:43:48,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9290 sec 2024-12-09T20:43:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T20:43:49,101 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:49,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-09T20:43:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T20:43:49,105 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:49,105 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:49,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:49,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:43:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:49,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:49,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:49,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:49,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:49,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/bcfad107158441359a3d2efcf04ed017 is 50, key is test_row_0/A:col10/1733777029181/Put/seqid=0 2024-12-09T20:43:49,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742263_1439 (size=19021) 2024-12-09T20:43:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T20:43:49,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777029207, max timestamp=1733777029218Num uniques 859; 2024-12-09T20:43:49,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777029207, max timestamp=1733777029218Num uniques 859; 2024-12-09T20:43:49,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:49,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:49,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733777029207, max timestamp=1733777029218Num uniques 859; 2024-12-09T20:43:49,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T20:43:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777089244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777089245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777089252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T20:43:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:49,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777089253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777089254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777089361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777089361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777089362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777089371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777089371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T20:43:49,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:49,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:49,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T20:43:49,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T20:43:49,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:49,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:49,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:49,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777089572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777089572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777089572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/bcfad107158441359a3d2efcf04ed017 2024-12-09T20:43:49,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777089591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777089591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/8d888e2b66e64983a0e1972bca629285 is 50, key is test_row_0/B:col10/1733777029181/Put/seqid=0 2024-12-09T20:43:49,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742264_1440 (size=12001) 2024-12-09T20:43:49,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/8d888e2b66e64983a0e1972bca629285 2024-12-09T20:43:49,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/61709df56b9b4a3697e27e8d1ca281ee is 50, key is test_row_0/C:col10/1733777029181/Put/seqid=0 2024-12-09T20:43:49,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742265_1441 (size=12001) 2024-12-09T20:43:49,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/61709df56b9b4a3697e27e8d1ca281ee 2024-12-09T20:43:49,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/bcfad107158441359a3d2efcf04ed017 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bcfad107158441359a3d2efcf04ed017 2024-12-09T20:43:49,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bcfad107158441359a3d2efcf04ed017, entries=300, sequenceid=44, filesize=18.6 K 2024-12-09T20:43:49,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/8d888e2b66e64983a0e1972bca629285 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8d888e2b66e64983a0e1972bca629285 2024-12-09T20:43:49,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8d888e2b66e64983a0e1972bca629285, entries=150, sequenceid=44, filesize=11.7 K 2024-12-09T20:43:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/61709df56b9b4a3697e27e8d1ca281ee as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/61709df56b9b4a3697e27e8d1ca281ee 2024-12-09T20:43:49,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/61709df56b9b4a3697e27e8d1ca281ee, entries=150, sequenceid=44, filesize=11.7 K 2024-12-09T20:43:49,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 879ac9ed3549bd954296156f21c08ad4 in 491ms, sequenceid=44, compaction requested=true 2024-12-09T20:43:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:49,675 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:49,675 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:49,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:49,677 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:49,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:49,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:49,677 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/A is initiating minor compaction (all files) 2024-12-09T20:43:49,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:49,677 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/A in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,677 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/e598b860d9054457b86f3bfbf2ad98ff, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/7ac343aa9389497fba2d527f6b39b717, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bcfad107158441359a3d2efcf04ed017] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=44.3 K 2024-12-09T20:43:49,677 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:49,677 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/B is initiating minor compaction (all files) 2024-12-09T20:43:49,677 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/B in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,677 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/20f7e61568ec4ce1a6ee8bbc61bc3e23, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/23278f6d89ff483d95957f8ad290e370, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8d888e2b66e64983a0e1972bca629285] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=35.2 K 2024-12-09T20:43:49,678 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting e598b860d9054457b86f3bfbf2ad98ff, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733777026957 2024-12-09T20:43:49,678 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f7e61568ec4ce1a6ee8bbc61bc3e23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733777026957 2024-12-09T20:43:49,678 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ac343aa9389497fba2d527f6b39b717, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733777026988 2024-12-09T20:43:49,678 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 23278f6d89ff483d95957f8ad290e370, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733777026988 2024-12-09T20:43:49,678 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcfad107158441359a3d2efcf04ed017, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733777029153 2024-12-09T20:43:49,678 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d888e2b66e64983a0e1972bca629285, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733777029181 2024-12-09T20:43:49,685 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#A#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:49,685 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d0a09c1e19c44c3592b82f850fa51d24 is 50, key is test_row_0/A:col10/1733777029181/Put/seqid=0 2024-12-09T20:43:49,686 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#B#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:49,687 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/379b569203f1445c9c57d8da5c6503c3 is 50, key is test_row_0/B:col10/1733777029181/Put/seqid=0 2024-12-09T20:43:49,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742267_1443 (size=12104) 2024-12-09T20:43:49,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742266_1442 (size=12104) 2024-12-09T20:43:49,707 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/379b569203f1445c9c57d8da5c6503c3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/379b569203f1445c9c57d8da5c6503c3 2024-12-09T20:43:49,713 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/B of 879ac9ed3549bd954296156f21c08ad4 into 379b569203f1445c9c57d8da5c6503c3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:49,713 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:49,713 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/B, priority=13, startTime=1733777029675; duration=0sec 2024-12-09T20:43:49,713 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:49,713 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:B 2024-12-09T20:43:49,713 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:49,715 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:49,715 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/C is initiating minor compaction (all files) 2024-12-09T20:43:49,715 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/C in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,715 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7b8e047cab444a68ad7605eb4b28a644, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/eb054c3dce9e42a195de1ce4357ee3e4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/61709df56b9b4a3697e27e8d1ca281ee] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=35.2 K 2024-12-09T20:43:49,724 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b8e047cab444a68ad7605eb4b28a644, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733777026957 2024-12-09T20:43:49,726 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting eb054c3dce9e42a195de1ce4357ee3e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733777026988 2024-12-09T20:43:49,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T20:43:49,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:49,726 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-12-09T20:43:49,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:49,728 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 61709df56b9b4a3697e27e8d1ca281ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733777029181 2024-12-09T20:43:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T20:43:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/dc85c47db73b402f9f9de461445c414f is 50, key is test_row_0/A:col10/1733777029251/Put/seqid=0 2024-12-09T20:43:49,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742268_1444 (size=12001) 2024-12-09T20:43:49,754 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/dc85c47db73b402f9f9de461445c414f 2024-12-09T20:43:49,758 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:49,759 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/6e73a3d4a66142bd84ad665331006abc is 50, key is test_row_0/C:col10/1733777029181/Put/seqid=0 2024-12-09T20:43:49,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/6ab1afda55884a4e9a7c51ae4cc60c35 is 50, key is test_row_0/B:col10/1733777029251/Put/seqid=0 2024-12-09T20:43:49,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742269_1445 (size=12104) 2024-12-09T20:43:49,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742270_1446 (size=12001) 2024-12-09T20:43:49,829 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:43:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:49,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:49,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777089910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777089911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777089911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777089917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:49,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777089918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777090021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777090031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777090041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,106 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d0a09c1e19c44c3592b82f850fa51d24 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d0a09c1e19c44c3592b82f850fa51d24 2024-12-09T20:43:50,109 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/A of 879ac9ed3549bd954296156f21c08ad4 into d0a09c1e19c44c3592b82f850fa51d24(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:50,109 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:50,109 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/A, priority=13, startTime=1733777029675; duration=0sec 2024-12-09T20:43:50,109 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:50,109 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:A 2024-12-09T20:43:50,180 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/6e73a3d4a66142bd84ad665331006abc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6e73a3d4a66142bd84ad665331006abc 2024-12-09T20:43:50,182 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/C of 879ac9ed3549bd954296156f21c08ad4 into 6e73a3d4a66142bd84ad665331006abc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:50,183 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:50,183 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/C, priority=13, startTime=1733777029677; duration=0sec 2024-12-09T20:43:50,183 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:50,183 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:C 2024-12-09T20:43:50,186 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/6ab1afda55884a4e9a7c51ae4cc60c35 2024-12-09T20:43:50,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/6a5531f4d6f744ea8ccc74a6e94374bb is 50, key is test_row_0/C:col10/1733777029251/Put/seqid=0 2024-12-09T20:43:50,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742271_1447 (size=12001) 2024-12-09T20:43:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T20:43:50,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777090242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777090251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777090251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777090431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777090431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777090550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777090561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777090561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:50,601 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/6a5531f4d6f744ea8ccc74a6e94374bb 2024-12-09T20:43:50,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/dc85c47db73b402f9f9de461445c414f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dc85c47db73b402f9f9de461445c414f 2024-12-09T20:43:50,611 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dc85c47db73b402f9f9de461445c414f, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T20:43:50,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/6ab1afda55884a4e9a7c51ae4cc60c35 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/6ab1afda55884a4e9a7c51ae4cc60c35 2024-12-09T20:43:50,615 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/6ab1afda55884a4e9a7c51ae4cc60c35, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T20:43:50,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/6a5531f4d6f744ea8ccc74a6e94374bb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6a5531f4d6f744ea8ccc74a6e94374bb 2024-12-09T20:43:50,618 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6a5531f4d6f744ea8ccc74a6e94374bb, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T20:43:50,619 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=60.38 KB/61830 for 879ac9ed3549bd954296156f21c08ad4 in 893ms, sequenceid=81, compaction requested=false 2024-12-09T20:43:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-09T20:43:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-09T20:43:50,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-09T20:43:50,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5150 sec 2024-12-09T20:43:50,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5210 sec 2024-12-09T20:43:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:51,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:43:51,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:51,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:51,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:51,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:51,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:51,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:51,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/051c1e27463f4c6ea0405a710a8d5512 is 50, key is test_row_0/A:col10/1733777029902/Put/seqid=0 2024-12-09T20:43:51,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742272_1448 (size=14341) 2024-12-09T20:43:51,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/051c1e27463f4c6ea0405a710a8d5512 2024-12-09T20:43:51,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/9924f32b2baf4515bc1d6687498e6a5e is 50, key is test_row_0/B:col10/1733777029902/Put/seqid=0 2024-12-09T20:43:51,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742273_1449 (size=12001) 2024-12-09T20:43:51,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/9924f32b2baf4515bc1d6687498e6a5e 2024-12-09T20:43:51,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/3bea9972ee564c4d963772d62386fd09 is 50, key is test_row_0/C:col10/1733777029902/Put/seqid=0 2024-12-09T20:43:51,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742274_1450 (size=12001) 2024-12-09T20:43:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777091118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/3bea9972ee564c4d963772d62386fd09 2024-12-09T20:43:51,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777091119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777091119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/051c1e27463f4c6ea0405a710a8d5512 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/051c1e27463f4c6ea0405a710a8d5512 2024-12-09T20:43:51,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/051c1e27463f4c6ea0405a710a8d5512, entries=200, sequenceid=96, filesize=14.0 K 2024-12-09T20:43:51,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/9924f32b2baf4515bc1d6687498e6a5e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9924f32b2baf4515bc1d6687498e6a5e 2024-12-09T20:43:51,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9924f32b2baf4515bc1d6687498e6a5e, entries=150, sequenceid=96, filesize=11.7 K 2024-12-09T20:43:51,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/3bea9972ee564c4d963772d62386fd09 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3bea9972ee564c4d963772d62386fd09 2024-12-09T20:43:51,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3bea9972ee564c4d963772d62386fd09, entries=150, sequenceid=96, filesize=11.7 K 2024-12-09T20:43:51,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 879ac9ed3549bd954296156f21c08ad4 in 99ms, sequenceid=96, compaction requested=true 2024-12-09T20:43:51,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:51,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:51,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:51,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:51,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:43:51,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:51,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-09T20:43:51,164 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:51,165 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:51,176 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:51,176 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:51,176 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/A is initiating minor compaction (all files) 2024-12-09T20:43:51,176 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/C is initiating minor compaction (all files) 2024-12-09T20:43:51,176 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/C in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,176 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/A in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,177 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d0a09c1e19c44c3592b82f850fa51d24, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dc85c47db73b402f9f9de461445c414f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/051c1e27463f4c6ea0405a710a8d5512] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=37.5 K 2024-12-09T20:43:51,177 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6e73a3d4a66142bd84ad665331006abc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6a5531f4d6f744ea8ccc74a6e94374bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3bea9972ee564c4d963772d62386fd09] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=35.3 K 2024-12-09T20:43:51,177 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a09c1e19c44c3592b82f850fa51d24, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733777029181 2024-12-09T20:43:51,177 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e73a3d4a66142bd84ad665331006abc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733777029181 2024-12-09T20:43:51,177 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc85c47db73b402f9f9de461445c414f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733777029238 2024-12-09T20:43:51,177 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a5531f4d6f744ea8ccc74a6e94374bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733777029238 2024-12-09T20:43:51,177 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 051c1e27463f4c6ea0405a710a8d5512, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733777029902 2024-12-09T20:43:51,177 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bea9972ee564c4d963772d62386fd09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733777029902 2024-12-09T20:43:51,187 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#C#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:51,188 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#A#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:51,188 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/97e66addc1d0436d936b3acdd2d063c3 is 50, key is test_row_0/C:col10/1733777029902/Put/seqid=0 2024-12-09T20:43:51,188 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/277f22b0cda4428ab897ba4b5b82aaba is 50, key is test_row_0/A:col10/1733777029902/Put/seqid=0 2024-12-09T20:43:51,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742276_1452 (size=12207) 2024-12-09T20:43:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742275_1451 (size=12207) 2024-12-09T20:43:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:51,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:43:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:51,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/dd04056ebd0c4c7bb77f05818fbc611f is 50, key is test_row_0/A:col10/1733777031234/Put/seqid=0 2024-12-09T20:43:51,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742277_1453 (size=14341) 2024-12-09T20:43:51,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/dd04056ebd0c4c7bb77f05818fbc611f 2024-12-09T20:43:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T20:43:51,255 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:51,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:51,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/ab8594d2935941919567c76201d11d85 is 50, key is test_row_0/B:col10/1733777031234/Put/seqid=0 2024-12-09T20:43:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-09T20:43:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T20:43:51,258 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:51,259 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:51,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777091262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777091271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742278_1454 (size=12001) 2024-12-09T20:43:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777091274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T20:43:51,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777091381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777091381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777091391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T20:43:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:51,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:51,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:51,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777091441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,452 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777091452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,455 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T20:43:51,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:51,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:51,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T20:43:51,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777091591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777091591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,606 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/277f22b0cda4428ab897ba4b5b82aaba as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/277f22b0cda4428ab897ba4b5b82aaba 2024-12-09T20:43:51,607 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/97e66addc1d0436d936b3acdd2d063c3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/97e66addc1d0436d936b3acdd2d063c3 2024-12-09T20:43:51,610 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/A of 879ac9ed3549bd954296156f21c08ad4 into 277f22b0cda4428ab897ba4b5b82aaba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:51,610 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:51,610 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/A, priority=13, startTime=1733777031163; duration=0sec 2024-12-09T20:43:51,610 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:51,610 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:A 2024-12-09T20:43:51,610 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:51,613 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:51,613 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/B is initiating minor compaction (all files) 2024-12-09T20:43:51,613 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/B in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,613 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/379b569203f1445c9c57d8da5c6503c3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/6ab1afda55884a4e9a7c51ae4cc60c35, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9924f32b2baf4515bc1d6687498e6a5e] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=35.3 K 2024-12-09T20:43:51,615 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 379b569203f1445c9c57d8da5c6503c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733777029181 2024-12-09T20:43:51,615 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/C of 879ac9ed3549bd954296156f21c08ad4 into 97e66addc1d0436d936b3acdd2d063c3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:51,615 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:51,615 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/C, priority=13, startTime=1733777031163; duration=0sec 2024-12-09T20:43:51,615 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:51,615 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:C 2024-12-09T20:43:51,615 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab1afda55884a4e9a7c51ae4cc60c35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733777029238 2024-12-09T20:43:51,617 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9924f32b2baf4515bc1d6687498e6a5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733777029902 2024-12-09T20:43:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777091611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,635 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:51,635 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/a9012c2a394449cbbbef9c6a1f8f8aed is 50, key is test_row_0/B:col10/1733777029902/Put/seqid=0 2024-12-09T20:43:51,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742279_1455 (size=12207) 2024-12-09T20:43:51,665 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/a9012c2a394449cbbbef9c6a1f8f8aed as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/a9012c2a394449cbbbef9c6a1f8f8aed 2024-12-09T20:43:51,670 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/B of 879ac9ed3549bd954296156f21c08ad4 into a9012c2a394449cbbbef9c6a1f8f8aed(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:51,670 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:51,670 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/B, priority=13, startTime=1733777031163; duration=0sec 2024-12-09T20:43:51,670 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:51,670 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:B 2024-12-09T20:43:51,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/ab8594d2935941919567c76201d11d85 2024-12-09T20:43:51,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/29bdb486480f4f44be8bffa93333f4ce is 50, key is test_row_0/C:col10/1733777031234/Put/seqid=0 2024-12-09T20:43:51,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742280_1456 (size=12001) 2024-12-09T20:43:51,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T20:43:51,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:51,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:51,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:51,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/29bdb486480f4f44be8bffa93333f4ce 2024-12-09T20:43:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:51,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/dd04056ebd0c4c7bb77f05818fbc611f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dd04056ebd0c4c7bb77f05818fbc611f 2024-12-09T20:43:51,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dd04056ebd0c4c7bb77f05818fbc611f, entries=200, sequenceid=116, filesize=14.0 K 2024-12-09T20:43:51,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/ab8594d2935941919567c76201d11d85 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ab8594d2935941919567c76201d11d85 2024-12-09T20:43:51,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ab8594d2935941919567c76201d11d85, entries=150, sequenceid=116, filesize=11.7 K 2024-12-09T20:43:51,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/29bdb486480f4f44be8bffa93333f4ce as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/29bdb486480f4f44be8bffa93333f4ce 2024-12-09T20:43:51,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/29bdb486480f4f44be8bffa93333f4ce, entries=150, sequenceid=116, filesize=11.7 K 2024-12-09T20:43:51,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 879ac9ed3549bd954296156f21c08ad4 in 504ms, sequenceid=116, compaction requested=false 2024-12-09T20:43:51,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:51,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T20:43:51,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:51,873 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:43:51,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:51,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:51,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:51,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:51,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:51,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T20:43:51,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/727b0c9954f34a24bd6889ce18b3ca52 is 50, key is test_row_0/A:col10/1733777031258/Put/seqid=0 2024-12-09T20:43:51,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742281_1457 (size=12001) 2024-12-09T20:43:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:51,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:51,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777091950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777091951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:51,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777091958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777092061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777092071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:52,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777092071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777092271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777092281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777092281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,298 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/727b0c9954f34a24bd6889ce18b3ca52 2024-12-09T20:43:52,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/dceed8f57ba2448ea6c351ec91498219 is 50, key is test_row_0/B:col10/1733777031258/Put/seqid=0 2024-12-09T20:43:52,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742282_1458 (size=12001) 2024-12-09T20:43:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T20:43:52,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777092581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777092591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:52,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777092592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:52,709 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/dceed8f57ba2448ea6c351ec91498219 2024-12-09T20:43:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/5dd9dc0548504748befc7a9eff8182c5 is 50, key is test_row_0/C:col10/1733777031258/Put/seqid=0 2024-12-09T20:43:52,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742283_1459 (size=12001) 2024-12-09T20:43:52,743 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/5dd9dc0548504748befc7a9eff8182c5 2024-12-09T20:43:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/727b0c9954f34a24bd6889ce18b3ca52 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/727b0c9954f34a24bd6889ce18b3ca52 2024-12-09T20:43:52,760 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/727b0c9954f34a24bd6889ce18b3ca52, entries=150, sequenceid=131, filesize=11.7 K 2024-12-09T20:43:52,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/dceed8f57ba2448ea6c351ec91498219 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/dceed8f57ba2448ea6c351ec91498219 2024-12-09T20:43:52,766 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/dceed8f57ba2448ea6c351ec91498219, entries=150, sequenceid=131, filesize=11.7 K 2024-12-09T20:43:52,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/5dd9dc0548504748befc7a9eff8182c5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/5dd9dc0548504748befc7a9eff8182c5 2024-12-09T20:43:52,769 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/5dd9dc0548504748befc7a9eff8182c5, entries=150, sequenceid=131, filesize=11.7 K 2024-12-09T20:43:52,770 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 879ac9ed3549bd954296156f21c08ad4 in 898ms, sequenceid=131, compaction requested=true 2024-12-09T20:43:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-09T20:43:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-09T20:43:52,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-09T20:43:52,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5120 sec 2024-12-09T20:43:52,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5170 sec 2024-12-09T20:43:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:53,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T20:43:53,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:53,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:53,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:53,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:53,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:53,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:53,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d9bf2d43bd734123859eda77f9d4d840 is 50, key is test_row_0/A:col10/1733777031956/Put/seqid=0 2024-12-09T20:43:53,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742284_1460 (size=14541) 2024-12-09T20:43:53,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777093131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777093132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777093132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777093252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777093252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777093252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T20:43:53,401 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:53,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-09T20:43:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T20:43:53,405 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:53,405 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:53,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:53,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777093461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777093461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777093461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777093471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,476 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:53,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777093481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,485 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:43:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T20:43:53,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d9bf2d43bd734123859eda77f9d4d840 2024-12-09T20:43:53,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/89e86d8548ab4082abfc3ae7a66c0f4c is 50, key is test_row_0/B:col10/1733777031956/Put/seqid=0 2024-12-09T20:43:53,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742285_1461 (size=12151) 2024-12-09T20:43:53,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/89e86d8548ab4082abfc3ae7a66c0f4c 2024-12-09T20:43:53,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/81d1e07f0c5e40eeaa39a2bb346787e2 is 50, key is test_row_0/C:col10/1733777031956/Put/seqid=0 2024-12-09T20:43:53,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T20:43:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:53,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:53,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742286_1462 (size=12151) 2024-12-09T20:43:53,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T20:43:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:53,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T20:43:53,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777093771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777093771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777093771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:53,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T20:43:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:53,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:53,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/81d1e07f0c5e40eeaa39a2bb346787e2 2024-12-09T20:43:53,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d9bf2d43bd734123859eda77f9d4d840 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d9bf2d43bd734123859eda77f9d4d840 2024-12-09T20:43:53,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d9bf2d43bd734123859eda77f9d4d840, entries=200, sequenceid=152, filesize=14.2 K 2024-12-09T20:43:53,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/89e86d8548ab4082abfc3ae7a66c0f4c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/89e86d8548ab4082abfc3ae7a66c0f4c 2024-12-09T20:43:53,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/89e86d8548ab4082abfc3ae7a66c0f4c, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T20:43:53,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/81d1e07f0c5e40eeaa39a2bb346787e2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/81d1e07f0c5e40eeaa39a2bb346787e2 2024-12-09T20:43:53,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/81d1e07f0c5e40eeaa39a2bb346787e2, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T20:43:53,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 879ac9ed3549bd954296156f21c08ad4 in 896ms, sequenceid=152, compaction requested=true 2024-12-09T20:43:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:53,995 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:53,996 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:53,996 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:53,996 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/A is initiating minor compaction (all files) 2024-12-09T20:43:53,997 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/A in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:53,997 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/277f22b0cda4428ab897ba4b5b82aaba, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dd04056ebd0c4c7bb77f05818fbc611f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/727b0c9954f34a24bd6889ce18b3ca52, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d9bf2d43bd734123859eda77f9d4d840] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=51.8 K 2024-12-09T20:43:53,997 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:53,997 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 277f22b0cda4428ab897ba4b5b82aaba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733777029902 2024-12-09T20:43:53,997 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/B is initiating minor compaction (all files) 2024-12-09T20:43:53,997 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/B in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:53,997 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/a9012c2a394449cbbbef9c6a1f8f8aed, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ab8594d2935941919567c76201d11d85, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/dceed8f57ba2448ea6c351ec91498219, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/89e86d8548ab4082abfc3ae7a66c0f4c] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=47.2 K 2024-12-09T20:43:53,998 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9012c2a394449cbbbef9c6a1f8f8aed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733777029902 2024-12-09T20:43:53,998 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting dd04056ebd0c4c7bb77f05818fbc611f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733777031104 2024-12-09T20:43:53,998 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab8594d2935941919567c76201d11d85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733777031115 2024-12-09T20:43:53,999 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 727b0c9954f34a24bd6889ce18b3ca52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733777031254 2024-12-09T20:43:53,999 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting dceed8f57ba2448ea6c351ec91498219, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733777031254 2024-12-09T20:43:53,999 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d9bf2d43bd734123859eda77f9d4d840, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733777031917 2024-12-09T20:43:53,999 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e86d8548ab4082abfc3ae7a66c0f4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733777031917 2024-12-09T20:43:54,015 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:54,015 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/c397380489544146ba682c2d7c6f95ed is 50, key is test_row_0/B:col10/1733777031956/Put/seqid=0 2024-12-09T20:43:54,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T20:43:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:54,018 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:43:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:54,022 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#A#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:54,023 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/29d3ed84cd9343b2a69de74dd5049265 is 50, key is test_row_0/A:col10/1733777031956/Put/seqid=0 2024-12-09T20:43:54,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/5b0514dd5e114db9824d902e4926c83c is 50, key is test_row_1/A:col10/1733777033130/Put/seqid=0 2024-12-09T20:43:54,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742287_1463 (size=12493) 2024-12-09T20:43:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T20:43:54,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742288_1464 (size=12493) 2024-12-09T20:43:54,036 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/c397380489544146ba682c2d7c6f95ed as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c397380489544146ba682c2d7c6f95ed 2024-12-09T20:43:54,040 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/29d3ed84cd9343b2a69de74dd5049265 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/29d3ed84cd9343b2a69de74dd5049265 2024-12-09T20:43:54,040 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/B of 879ac9ed3549bd954296156f21c08ad4 into c397380489544146ba682c2d7c6f95ed(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:54,040 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:54,040 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/B, priority=12, startTime=1733777033996; duration=0sec 2024-12-09T20:43:54,040 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:54,040 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:B 2024-12-09T20:43:54,040 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:43:54,045 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:43:54,045 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/C is initiating minor compaction (all files) 2024-12-09T20:43:54,045 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/C in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:54,045 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/97e66addc1d0436d936b3acdd2d063c3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/29bdb486480f4f44be8bffa93333f4ce, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/5dd9dc0548504748befc7a9eff8182c5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/81d1e07f0c5e40eeaa39a2bb346787e2] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=47.2 K 2024-12-09T20:43:54,045 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97e66addc1d0436d936b3acdd2d063c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733777029902 2024-12-09T20:43:54,046 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29bdb486480f4f44be8bffa93333f4ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733777031115 2024-12-09T20:43:54,046 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd9dc0548504748befc7a9eff8182c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733777031254 2024-12-09T20:43:54,046 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81d1e07f0c5e40eeaa39a2bb346787e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733777031917 2024-12-09T20:43:54,047 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/A of 879ac9ed3549bd954296156f21c08ad4 into 29d3ed84cd9343b2a69de74dd5049265(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:54,047 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:54,047 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/A, priority=12, startTime=1733777033995; duration=0sec 2024-12-09T20:43:54,048 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:54,048 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:A 2024-12-09T20:43:54,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742289_1465 (size=9757) 2024-12-09T20:43:54,073 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:54,073 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/4aeb3813378f4573b603b3a1cae10f90 is 50, key is test_row_0/C:col10/1733777031956/Put/seqid=0 2024-12-09T20:43:54,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742290_1466 (size=12493) 2024-12-09T20:43:54,083 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/4aeb3813378f4573b603b3a1cae10f90 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/4aeb3813378f4573b603b3a1cae10f90 2024-12-09T20:43:54,089 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/C of 879ac9ed3549bd954296156f21c08ad4 into 4aeb3813378f4573b603b3a1cae10f90(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:54,089 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:54,089 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/C, priority=12, startTime=1733777033996; duration=0sec 2024-12-09T20:43:54,089 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:54,089 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:C 2024-12-09T20:43:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:54,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:54,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777094343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:54,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777094345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777094346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:54,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777094451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:54,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777094461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777094461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:54,467 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/5b0514dd5e114db9824d902e4926c83c 2024-12-09T20:43:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:54,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/f2cd90889a6e41e194d7f3aa136a4a74 is 50, key is test_row_1/B:col10/1733777033130/Put/seqid=0 2024-12-09T20:43:54,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742291_1467 (size=9757) 2024-12-09T20:43:54,480 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/f2cd90889a6e41e194d7f3aa136a4a74 2024-12-09T20:43:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/7501782285314e569e5bb4e51612273f is 50, key is test_row_1/C:col10/1733777033130/Put/seqid=0 2024-12-09T20:43:54,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742292_1468 (size=9757) 2024-12-09T20:43:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T20:43:54,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777094661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:54,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777094671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777094671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:54,895 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/7501782285314e569e5bb4e51612273f 2024-12-09T20:43:54,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/5b0514dd5e114db9824d902e4926c83c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/5b0514dd5e114db9824d902e4926c83c 2024-12-09T20:43:54,900 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/5b0514dd5e114db9824d902e4926c83c, entries=100, sequenceid=164, filesize=9.5 K 2024-12-09T20:43:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/f2cd90889a6e41e194d7f3aa136a4a74 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/f2cd90889a6e41e194d7f3aa136a4a74 2024-12-09T20:43:54,903 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/f2cd90889a6e41e194d7f3aa136a4a74, entries=100, sequenceid=164, filesize=9.5 K 2024-12-09T20:43:54,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/7501782285314e569e5bb4e51612273f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7501782285314e569e5bb4e51612273f 2024-12-09T20:43:54,906 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7501782285314e569e5bb4e51612273f, entries=100, sequenceid=164, filesize=9.5 K 2024-12-09T20:43:54,907 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 879ac9ed3549bd954296156f21c08ad4 in 889ms, sequenceid=164, compaction requested=false 2024-12-09T20:43:54,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:54,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:54,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-09T20:43:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-09T20:43:54,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-09T20:43:54,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-12-09T20:43:54,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5090 sec 2024-12-09T20:43:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:54,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T20:43:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:54,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d1b72afea42d4a078a9d394e05b2e1d6 is 50, key is test_row_0/A:col10/1733777034973/Put/seqid=0 2024-12-09T20:43:54,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742293_1469 (size=14541) 2024-12-09T20:43:54,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777094997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777094998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777094998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777095101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777095111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777095111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777095311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777095321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777095321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T20:43:55,372 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T20:43:55,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d1b72afea42d4a078a9d394e05b2e1d6 2024-12-09T20:43:55,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/454697af872a45128cdb16c6d373999a is 50, key is test_row_0/B:col10/1733777034973/Put/seqid=0 2024-12-09T20:43:55,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742294_1470 (size=12151) 2024-12-09T20:43:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T20:43:55,551 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:55,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-09T20:43:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T20:43:55,553 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:55,553 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:55,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:55,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777095621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777095631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777095632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T20:43:55,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T20:43:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:55,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:55,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/454697af872a45128cdb16c6d373999a 2024-12-09T20:43:55,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/38ce96d0181745abb8dd909de1816c12 is 50, key is test_row_0/C:col10/1733777034973/Put/seqid=0 2024-12-09T20:43:55,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742295_1471 (size=12151) 2024-12-09T20:43:55,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/38ce96d0181745abb8dd909de1816c12 2024-12-09T20:43:55,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d1b72afea42d4a078a9d394e05b2e1d6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d1b72afea42d4a078a9d394e05b2e1d6 2024-12-09T20:43:55,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d1b72afea42d4a078a9d394e05b2e1d6, entries=200, sequenceid=188, filesize=14.2 K 2024-12-09T20:43:55,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/454697af872a45128cdb16c6d373999a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/454697af872a45128cdb16c6d373999a 2024-12-09T20:43:55,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/454697af872a45128cdb16c6d373999a, entries=150, sequenceid=188, filesize=11.9 K 2024-12-09T20:43:55,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/38ce96d0181745abb8dd909de1816c12 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/38ce96d0181745abb8dd909de1816c12 2024-12-09T20:43:55,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/38ce96d0181745abb8dd909de1816c12, entries=150, sequenceid=188, filesize=11.9 K 2024-12-09T20:43:55,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 879ac9ed3549bd954296156f21c08ad4 in 858ms, sequenceid=188, compaction requested=true 2024-12-09T20:43:55,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:55,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:55,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:55,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:55,833 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:55,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:55,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:55,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:43:55,833 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:55,836 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:55,836 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:55,836 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/A is initiating minor compaction (all files) 2024-12-09T20:43:55,836 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/B is initiating minor compaction (all files) 2024-12-09T20:43:55,836 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/B in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:55,836 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/A in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:55,836 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c397380489544146ba682c2d7c6f95ed, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/f2cd90889a6e41e194d7f3aa136a4a74, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/454697af872a45128cdb16c6d373999a] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=33.6 K 2024-12-09T20:43:55,836 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/29d3ed84cd9343b2a69de74dd5049265, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/5b0514dd5e114db9824d902e4926c83c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d1b72afea42d4a078a9d394e05b2e1d6] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=35.9 K 2024-12-09T20:43:55,836 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c397380489544146ba682c2d7c6f95ed, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733777031917 2024-12-09T20:43:55,836 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d3ed84cd9343b2a69de74dd5049265, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733777031917 2024-12-09T20:43:55,837 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f2cd90889a6e41e194d7f3aa136a4a74, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733777033130 2024-12-09T20:43:55,837 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b0514dd5e114db9824d902e4926c83c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733777033130 2024-12-09T20:43:55,839 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 454697af872a45128cdb16c6d373999a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733777034336 2024-12-09T20:43:55,839 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b72afea42d4a078a9d394e05b2e1d6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733777034336 2024-12-09T20:43:55,843 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#A#compaction#399 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:55,844 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/42146be482ab454eb0d30494415f9784 is 50, key is test_row_0/A:col10/1733777034973/Put/seqid=0 2024-12-09T20:43:55,854 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:55,855 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/bef055b308c44375bd0958b944f00927 is 50, key is test_row_0/B:col10/1733777034973/Put/seqid=0 2024-12-09T20:43:55,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T20:43:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:55,859 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T20:43:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T20:43:55,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742296_1472 (size=12595) 2024-12-09T20:43:55,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/4a7ec5331ece43a2b23d1ad58937ba57 is 50, key is test_row_0/A:col10/1733777034986/Put/seqid=0 2024-12-09T20:43:55,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742297_1473 (size=12595) 2024-12-09T20:43:55,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742298_1474 (size=12151) 2024-12-09T20:43:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:56,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T20:43:56,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777096178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733777036162, max timestamp=1733777036181Num uniques 859; 2024-12-09T20:43:56,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:43:56,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733777036162, max timestamp=1733777036181Num uniques 859; 2024-12-09T20:43:56,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733777036162, max timestamp=1733777036181Num uniques 859; 2024-12-09T20:43:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:43:56,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:43:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777096246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777096247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,274 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/42146be482ab454eb0d30494415f9784 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/42146be482ab454eb0d30494415f9784 2024-12-09T20:43:56,277 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/A of 879ac9ed3549bd954296156f21c08ad4 into 42146be482ab454eb0d30494415f9784(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:56,277 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:56,277 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/A, priority=13, startTime=1733777035833; duration=0sec 2024-12-09T20:43:56,277 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:56,277 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:A 2024-12-09T20:43:56,278 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:56,278 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:56,278 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/C is initiating minor compaction (all files) 2024-12-09T20:43:56,278 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/C in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:56,278 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/4aeb3813378f4573b603b3a1cae10f90, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7501782285314e569e5bb4e51612273f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/38ce96d0181745abb8dd909de1816c12] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=33.6 K 2024-12-09T20:43:56,279 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aeb3813378f4573b603b3a1cae10f90, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733777031917 2024-12-09T20:43:56,279 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7501782285314e569e5bb4e51612273f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733777033130 2024-12-09T20:43:56,279 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38ce96d0181745abb8dd909de1816c12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733777034336 2024-12-09T20:43:56,285 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#C#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:56,286 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/342f25e66776499191db7cf5f1000319 is 50, key is test_row_0/C:col10/1733777034973/Put/seqid=0 2024-12-09T20:43:56,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742299_1475 (size=12595) 2024-12-09T20:43:56,292 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/bef055b308c44375bd0958b944f00927 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/bef055b308c44375bd0958b944f00927 2024-12-09T20:43:56,295 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/B of 879ac9ed3549bd954296156f21c08ad4 into bef055b308c44375bd0958b944f00927(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:56,295 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:56,295 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/B, priority=13, startTime=1733777035833; duration=0sec 2024-12-09T20:43:56,295 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:56,295 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:B 2024-12-09T20:43:56,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777096291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,298 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/4a7ec5331ece43a2b23d1ad58937ba57 2024-12-09T20:43:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/0c3f0875c4d54e88a97ac48099b4c158 is 50, key is test_row_0/B:col10/1733777034986/Put/seqid=0 2024-12-09T20:43:56,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742300_1476 (size=12151) 2024-12-09T20:43:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777096351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777096361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777096501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777096561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777096571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T20:43:56,694 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/342f25e66776499191db7cf5f1000319 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/342f25e66776499191db7cf5f1000319 2024-12-09T20:43:56,703 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/C of 879ac9ed3549bd954296156f21c08ad4 into 342f25e66776499191db7cf5f1000319(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:56,703 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:56,703 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/C, priority=13, startTime=1733777035833; duration=0sec 2024-12-09T20:43:56,703 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:56,703 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:C 2024-12-09T20:43:56,710 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/0c3f0875c4d54e88a97ac48099b4c158 2024-12-09T20:43:56,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/fdb4f47dabd744888d299dfac8d8035e is 50, key is test_row_0/C:col10/1733777034986/Put/seqid=0 2024-12-09T20:43:56,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742301_1477 (size=12151) 2024-12-09T20:43:56,723 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/fdb4f47dabd744888d299dfac8d8035e 2024-12-09T20:43:56,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/4a7ec5331ece43a2b23d1ad58937ba57 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4a7ec5331ece43a2b23d1ad58937ba57 2024-12-09T20:43:56,728 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4a7ec5331ece43a2b23d1ad58937ba57, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T20:43:56,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/0c3f0875c4d54e88a97ac48099b4c158 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/0c3f0875c4d54e88a97ac48099b4c158 2024-12-09T20:43:56,733 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/0c3f0875c4d54e88a97ac48099b4c158, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T20:43:56,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/fdb4f47dabd744888d299dfac8d8035e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/fdb4f47dabd744888d299dfac8d8035e 2024-12-09T20:43:56,739 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/fdb4f47dabd744888d299dfac8d8035e, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T20:43:56,740 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 879ac9ed3549bd954296156f21c08ad4 in 881ms, sequenceid=199, compaction requested=false 2024-12-09T20:43:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-09T20:43:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-09T20:43:56,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-09T20:43:56,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1880 sec 2024-12-09T20:43:56,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1940 sec 2024-12-09T20:43:56,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T20:43:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:56,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:56,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:56,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:56,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:56,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:56,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:56,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/0b28e8aed1784a71925a1a1470d1016e is 50, key is test_row_0/A:col10/1733777036246/Put/seqid=0 2024-12-09T20:43:56,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742302_1478 (size=14541) 2024-12-09T20:43:56,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/0b28e8aed1784a71925a1a1470d1016e 2024-12-09T20:43:56,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/ad7387fdaeae47ff9125158ba4fee025 is 50, key is test_row_0/B:col10/1733777036246/Put/seqid=0 2024-12-09T20:43:56,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742303_1479 (size=12151) 2024-12-09T20:43:56,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/ad7387fdaeae47ff9125158ba4fee025 2024-12-09T20:43:56,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/6d17c3e3213c42ac8651e7735276bca1 is 50, key is test_row_0/C:col10/1733777036246/Put/seqid=0 2024-12-09T20:43:56,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742304_1480 (size=12151) 2024-12-09T20:43:56,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777096881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777096882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777096884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:56,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777096991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777097001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777097211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777097211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/6d17c3e3213c42ac8651e7735276bca1 2024-12-09T20:43:57,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/0b28e8aed1784a71925a1a1470d1016e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/0b28e8aed1784a71925a1a1470d1016e 2024-12-09T20:43:57,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/0b28e8aed1784a71925a1a1470d1016e, entries=200, sequenceid=238, filesize=14.2 K 2024-12-09T20:43:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/ad7387fdaeae47ff9125158ba4fee025 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ad7387fdaeae47ff9125158ba4fee025 2024-12-09T20:43:57,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ad7387fdaeae47ff9125158ba4fee025, entries=150, sequenceid=238, filesize=11.9 K 2024-12-09T20:43:57,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/6d17c3e3213c42ac8651e7735276bca1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6d17c3e3213c42ac8651e7735276bca1 2024-12-09T20:43:57,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6d17c3e3213c42ac8651e7735276bca1, entries=150, sequenceid=238, filesize=11.9 K 2024-12-09T20:43:57,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 879ac9ed3549bd954296156f21c08ad4 in 491ms, sequenceid=238, compaction requested=true 2024-12-09T20:43:57,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:57,310 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:57,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:43:57,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:57,310 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:57,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:43:57,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:57,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:43:57,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:57,313 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:57,313 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/A is initiating minor compaction (all files) 2024-12-09T20:43:57,313 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/A in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:57,313 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/42146be482ab454eb0d30494415f9784, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4a7ec5331ece43a2b23d1ad58937ba57, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/0b28e8aed1784a71925a1a1470d1016e] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=38.4 K 2024-12-09T20:43:57,313 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42146be482ab454eb0d30494415f9784, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733777034336 2024-12-09T20:43:57,313 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a7ec5331ece43a2b23d1ad58937ba57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733777034986 2024-12-09T20:43:57,314 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b28e8aed1784a71925a1a1470d1016e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733777036201 2024-12-09T20:43:57,315 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:57,315 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/B is initiating minor compaction (all files) 2024-12-09T20:43:57,315 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/B in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:57,315 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/bef055b308c44375bd0958b944f00927, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/0c3f0875c4d54e88a97ac48099b4c158, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ad7387fdaeae47ff9125158ba4fee025] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=36.0 K 2024-12-09T20:43:57,316 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting bef055b308c44375bd0958b944f00927, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733777034336 2024-12-09T20:43:57,316 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c3f0875c4d54e88a97ac48099b4c158, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733777034986 2024-12-09T20:43:57,317 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting ad7387fdaeae47ff9125158ba4fee025, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733777036201 2024-12-09T20:43:57,322 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:57,322 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d90b77466a504e97a5166ebebba573e5 is 50, key is test_row_0/A:col10/1733777036246/Put/seqid=0 2024-12-09T20:43:57,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742305_1481 (size=12697) 2024-12-09T20:43:57,329 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#B#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:57,329 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/fc49d8eec913426ea7d4f70d5e298865 is 50, key is test_row_0/B:col10/1733777036246/Put/seqid=0 2024-12-09T20:43:57,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742306_1482 (size=12697) 2024-12-09T20:43:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:57,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T20:43:57,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:57,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:57,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:57,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:57,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:57,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:57,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/f08cf2b364394495bb1576f33e41459a is 50, key is test_row_0/A:col10/1733777036880/Put/seqid=0 2024-12-09T20:43:57,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742307_1483 (size=14541) 2024-12-09T20:43:57,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/f08cf2b364394495bb1576f33e41459a 2024-12-09T20:43:57,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/b40a2dbf3eb844a682ab373c98a090b1 is 50, key is test_row_0/B:col10/1733777036880/Put/seqid=0 2024-12-09T20:43:57,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742308_1484 (size=12151) 2024-12-09T20:43:57,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777037506, max timestamp=1733777037528Num uniques 859; 2024-12-09T20:43:57,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:57,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777037506, max timestamp=1733777037528Num uniques 859; 2024-12-09T20:43:57,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:43:57,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733777037506, max timestamp=1733777037527Num uniques 859; 2024-12-09T20:43:57,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T20:43:57,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777097565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777097566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777097566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777097568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777097568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777097681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777097681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777097681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777097681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777097681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T20:43:57,701 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:57,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-09T20:43:57,703 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:57,704 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:57,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T20:43:57,732 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d90b77466a504e97a5166ebebba573e5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d90b77466a504e97a5166ebebba573e5 2024-12-09T20:43:57,735 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/A of 879ac9ed3549bd954296156f21c08ad4 into d90b77466a504e97a5166ebebba573e5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:57,735 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:57,735 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/A, priority=13, startTime=1733777037310; duration=0sec 2024-12-09T20:43:57,735 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:43:57,735 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:A 2024-12-09T20:43:57,735 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:43:57,735 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:43:57,735 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/C is initiating minor compaction (all files) 2024-12-09T20:43:57,736 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/C in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:57,736 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/342f25e66776499191db7cf5f1000319, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/fdb4f47dabd744888d299dfac8d8035e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6d17c3e3213c42ac8651e7735276bca1] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=36.0 K 2024-12-09T20:43:57,736 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 342f25e66776499191db7cf5f1000319, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733777034336 2024-12-09T20:43:57,736 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdb4f47dabd744888d299dfac8d8035e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733777034986 2024-12-09T20:43:57,736 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d17c3e3213c42ac8651e7735276bca1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733777036201 2024-12-09T20:43:57,739 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/fc49d8eec913426ea7d4f70d5e298865 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/fc49d8eec913426ea7d4f70d5e298865 2024-12-09T20:43:57,742 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/B of 879ac9ed3549bd954296156f21c08ad4 into fc49d8eec913426ea7d4f70d5e298865(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:57,742 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:57,742 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/B, priority=13, startTime=1733777037310; duration=0sec 2024-12-09T20:43:57,742 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:57,742 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:B 2024-12-09T20:43:57,744 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#C#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:43:57,744 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/bf5aa2c80d9c4106a112295c9da5ecdb is 50, key is test_row_0/C:col10/1733777036246/Put/seqid=0 2024-12-09T20:43:57,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742309_1485 (size=12697) 2024-12-09T20:43:57,753 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/bf5aa2c80d9c4106a112295c9da5ecdb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bf5aa2c80d9c4106a112295c9da5ecdb 2024-12-09T20:43:57,766 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/C of 879ac9ed3549bd954296156f21c08ad4 into bf5aa2c80d9c4106a112295c9da5ecdb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:43:57,767 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:57,767 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/C, priority=13, startTime=1733777037312; duration=0sec 2024-12-09T20:43:57,767 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:43:57,767 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:C 2024-12-09T20:43:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T20:43:57,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/b40a2dbf3eb844a682ab373c98a090b1 2024-12-09T20:43:57,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T20:43:57,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:57,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:57,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:57,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:57,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:57,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/f23a9602ad5643f7b4648f561b13ddcc is 50, key is test_row_0/C:col10/1733777036880/Put/seqid=0 2024-12-09T20:43:57,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742310_1486 (size=12151) 2024-12-09T20:43:57,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777097891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777097891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777097891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777097892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777097892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T20:43:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:58,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:58,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:58,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T20:43:58,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T20:43:58,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:58,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:58,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:58,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:58,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:58,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:43:58,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777098201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777098201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777098202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777098202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777098211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/f23a9602ad5643f7b4648f561b13ddcc 2024-12-09T20:43:58,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/f08cf2b364394495bb1576f33e41459a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/f08cf2b364394495bb1576f33e41459a 2024-12-09T20:43:58,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/f08cf2b364394495bb1576f33e41459a, entries=200, sequenceid=249, filesize=14.2 K 2024-12-09T20:43:58,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/b40a2dbf3eb844a682ab373c98a090b1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/b40a2dbf3eb844a682ab373c98a090b1 2024-12-09T20:43:58,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/b40a2dbf3eb844a682ab373c98a090b1, entries=150, sequenceid=249, filesize=11.9 K 2024-12-09T20:43:58,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/f23a9602ad5643f7b4648f561b13ddcc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f23a9602ad5643f7b4648f561b13ddcc 2024-12-09T20:43:58,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f23a9602ad5643f7b4648f561b13ddcc, entries=150, sequenceid=249, filesize=11.9 K 2024-12-09T20:43:58,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 879ac9ed3549bd954296156f21c08ad4 in 879ms, sequenceid=249, compaction requested=false 2024-12-09T20:43:58,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:58,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T20:43:58,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:58,316 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-12-09T20:43:58,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:58,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:58,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:58,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:58,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:58,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:43:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/661216d5a1954754a8e14474aca3f598 is 50, key is test_row_0/A:col10/1733777037565/Put/seqid=0 2024-12-09T20:43:58,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742311_1487 (size=12301) 2024-12-09T20:43:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T20:43:58,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:43:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:58,723 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/661216d5a1954754a8e14474aca3f598 2024-12-09T20:43:58,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777098721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777098721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/54ccf6ead9f44dcc9f02c4878ca0a8fa is 50, key is test_row_0/B:col10/1733777037565/Put/seqid=0 2024-12-09T20:43:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777098726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777098728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777098729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742312_1488 (size=12301) 2024-12-09T20:43:58,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777098831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T20:43:58,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777098841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777098850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777098851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777099051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777099061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777099061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777099062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,145 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/54ccf6ead9f44dcc9f02c4878ca0a8fa 2024-12-09T20:43:59,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/582df2bd4efe42c69f19c159086e85bb is 50, key is test_row_0/C:col10/1733777037565/Put/seqid=0 2024-12-09T20:43:59,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742313_1489 (size=12301) 2024-12-09T20:43:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777099371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777099371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777099371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777099371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,556 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/582df2bd4efe42c69f19c159086e85bb 2024-12-09T20:43:59,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/661216d5a1954754a8e14474aca3f598 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/661216d5a1954754a8e14474aca3f598 2024-12-09T20:43:59,562 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/661216d5a1954754a8e14474aca3f598, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T20:43:59,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/54ccf6ead9f44dcc9f02c4878ca0a8fa as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/54ccf6ead9f44dcc9f02c4878ca0a8fa 2024-12-09T20:43:59,565 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/54ccf6ead9f44dcc9f02c4878ca0a8fa, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T20:43:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/582df2bd4efe42c69f19c159086e85bb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/582df2bd4efe42c69f19c159086e85bb 2024-12-09T20:43:59,571 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/582df2bd4efe42c69f19c159086e85bb, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T20:43:59,572 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=53.67 KB/54960 for 879ac9ed3549bd954296156f21c08ad4 in 1256ms, sequenceid=288, compaction requested=true 2024-12-09T20:43:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:43:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:43:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-09T20:43:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-09T20:43:59,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-09T20:43:59,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-12-09T20:43:59,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.8740 sec 2024-12-09T20:43:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:43:59,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:43:59,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:43:59,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:59,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:43:59,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:59,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:43:59,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:43:59,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/834a8b373f9f428aad1826e95ae57716 is 50, key is test_row_0/A:col10/1733777039742/Put/seqid=0 2024-12-09T20:43:59,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742314_1490 (size=9857) 2024-12-09T20:43:59,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/834a8b373f9f428aad1826e95ae57716 2024-12-09T20:43:59,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/9271db0377e343d29d39d6f944cdb689 is 50, key is test_row_0/B:col10/1733777039742/Put/seqid=0 2024-12-09T20:43:59,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742315_1491 (size=9857) 2024-12-09T20:43:59,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/9271db0377e343d29d39d6f944cdb689 2024-12-09T20:43:59,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/45eeb0e6c27e44a09b41fabd36218b74 is 50, key is test_row_0/C:col10/1733777039742/Put/seqid=0 2024-12-09T20:43:59,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742316_1492 (size=9857) 2024-12-09T20:43:59,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777099841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T20:43:59,851 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:43:59,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:43:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-09T20:43:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T20:43:59,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:43:59,855 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:43:59,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:43:59,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777099881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777099881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777099892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777099892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:43:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777099951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:43:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:43:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:43:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T20:44:00,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T20:44:00,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:00,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:00,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:00,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:00,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:00,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T20:44:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:00,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:00,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777100161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T20:44:00,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/45eeb0e6c27e44a09b41fabd36218b74 2024-12-09T20:44:00,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/834a8b373f9f428aad1826e95ae57716 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/834a8b373f9f428aad1826e95ae57716 2024-12-09T20:44:00,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/834a8b373f9f428aad1826e95ae57716, entries=100, sequenceid=300, filesize=9.6 K 2024-12-09T20:44:00,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/9271db0377e343d29d39d6f944cdb689 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9271db0377e343d29d39d6f944cdb689 2024-12-09T20:44:00,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9271db0377e343d29d39d6f944cdb689, entries=100, sequenceid=300, filesize=9.6 K 2024-12-09T20:44:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/45eeb0e6c27e44a09b41fabd36218b74 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/45eeb0e6c27e44a09b41fabd36218b74 2024-12-09T20:44:00,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/45eeb0e6c27e44a09b41fabd36218b74, entries=100, sequenceid=300, filesize=9.6 K 2024-12-09T20:44:00,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 879ac9ed3549bd954296156f21c08ad4 in 441ms, sequenceid=300, compaction requested=true 2024-12-09T20:44:00,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:00,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:44:00,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:00,184 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:44:00,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:44:00,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:00,184 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:44:00,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:44:00,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:00,185 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47006 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:44:00,185 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:44:00,185 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/B is initiating minor compaction (all files) 2024-12-09T20:44:00,185 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/A is initiating minor compaction (all files) 2024-12-09T20:44:00,185 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/A in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:00,185 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/B in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:00,185 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/fc49d8eec913426ea7d4f70d5e298865, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/b40a2dbf3eb844a682ab373c98a090b1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/54ccf6ead9f44dcc9f02c4878ca0a8fa, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9271db0377e343d29d39d6f944cdb689] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=45.9 K 2024-12-09T20:44:00,185 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d90b77466a504e97a5166ebebba573e5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/f08cf2b364394495bb1576f33e41459a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/661216d5a1954754a8e14474aca3f598, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/834a8b373f9f428aad1826e95ae57716] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=48.2 K 2024-12-09T20:44:00,186 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting fc49d8eec913426ea7d4f70d5e298865, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733777036201 2024-12-09T20:44:00,186 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting d90b77466a504e97a5166ebebba573e5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733777036201 2024-12-09T20:44:00,186 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b40a2dbf3eb844a682ab373c98a090b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733777036880 2024-12-09T20:44:00,186 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting f08cf2b364394495bb1576f33e41459a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733777036829 2024-12-09T20:44:00,186 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ccf6ead9f44dcc9f02c4878ca0a8fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733777037552 2024-12-09T20:44:00,186 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 661216d5a1954754a8e14474aca3f598, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733777037552 2024-12-09T20:44:00,186 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 9271db0377e343d29d39d6f944cdb689, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733777038727 2024-12-09T20:44:00,186 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834a8b373f9f428aad1826e95ae57716, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733777038727 2024-12-09T20:44:00,193 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:00,193 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:00,193 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/7c1c729ef340499793795b8bdc63145a is 50, key is test_row_0/B:col10/1733777039742/Put/seqid=0 2024-12-09T20:44:00,193 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/1578037193064f23a6f25fd1ad2a1bcf is 50, key is test_row_0/A:col10/1733777039742/Put/seqid=0 2024-12-09T20:44:00,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742317_1493 (size=12983) 2024-12-09T20:44:00,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742318_1494 (size=12983) 2024-12-09T20:44:00,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T20:44:00,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:00,311 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:44:00,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:44:00,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:00,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:44:00,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:00,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:44:00,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:00,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/db4986aceee84297aee5af1f67307384 is 50, key is test_row_0/A:col10/1733777039821/Put/seqid=0 2024-12-09T20:44:00,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742319_1495 (size=12301) 2024-12-09T20:44:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:00,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T20:44:00,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777100514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777100630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:00,711 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/1578037193064f23a6f25fd1ad2a1bcf as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/1578037193064f23a6f25fd1ad2a1bcf 2024-12-09T20:44:00,711 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/7c1c729ef340499793795b8bdc63145a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/7c1c729ef340499793795b8bdc63145a 2024-12-09T20:44:00,714 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/B of 879ac9ed3549bd954296156f21c08ad4 into 7c1c729ef340499793795b8bdc63145a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:00,714 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/A of 879ac9ed3549bd954296156f21c08ad4 into 1578037193064f23a6f25fd1ad2a1bcf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:00,714 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:00,714 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:00,714 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/B, priority=12, startTime=1733777040184; duration=0sec 2024-12-09T20:44:00,714 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/A, priority=12, startTime=1733777040184; duration=0sec 2024-12-09T20:44:00,714 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:00,714 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:00,714 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:B 2024-12-09T20:44:00,714 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:A 2024-12-09T20:44:00,714 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:44:00,715 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47006 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:44:00,715 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/C is initiating minor compaction (all files) 2024-12-09T20:44:00,715 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/C in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:00,715 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bf5aa2c80d9c4106a112295c9da5ecdb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f23a9602ad5643f7b4648f561b13ddcc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/582df2bd4efe42c69f19c159086e85bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/45eeb0e6c27e44a09b41fabd36218b74] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=45.9 K 2024-12-09T20:44:00,716 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5aa2c80d9c4106a112295c9da5ecdb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733777036201 2024-12-09T20:44:00,716 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting f23a9602ad5643f7b4648f561b13ddcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733777036880 2024-12-09T20:44:00,716 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 582df2bd4efe42c69f19c159086e85bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733777037552 2024-12-09T20:44:00,716 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 45eeb0e6c27e44a09b41fabd36218b74, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733777038727 2024-12-09T20:44:00,718 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/db4986aceee84297aee5af1f67307384 2024-12-09T20:44:00,721 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#C#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:00,721 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/f1b61dc71013490e9733b80082d3c046 is 50, key is test_row_0/C:col10/1733777039742/Put/seqid=0 2024-12-09T20:44:00,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/e9a233cb332c414cbc6e1135f6c26818 is 50, key is test_row_0/B:col10/1733777039821/Put/seqid=0 2024-12-09T20:44:00,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742320_1496 (size=12983) 2024-12-09T20:44:00,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742321_1497 (size=12301) 2024-12-09T20:44:00,727 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/f1b61dc71013490e9733b80082d3c046 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f1b61dc71013490e9733b80082d3c046 2024-12-09T20:44:00,731 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/C of 879ac9ed3549bd954296156f21c08ad4 into f1b61dc71013490e9733b80082d3c046(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:00,731 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:00,731 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/C, priority=12, startTime=1733777040184; duration=0sec 2024-12-09T20:44:00,731 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:00,731 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:C 2024-12-09T20:44:00,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777100841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:00,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777100890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:00,893 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:00,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777100901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:00,906 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:00,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777100911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:00,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777100911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:00,916 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:00,916 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T20:44:01,127 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/e9a233cb332c414cbc6e1135f6c26818 2024-12-09T20:44:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/0602ebcdf4e440b1a1db3580806255ab is 50, key is test_row_0/C:col10/1733777039821/Put/seqid=0 2024-12-09T20:44:01,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742322_1498 (size=12301) 2024-12-09T20:44:01,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777101151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:01,536 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/0602ebcdf4e440b1a1db3580806255ab 2024-12-09T20:44:01,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/db4986aceee84297aee5af1f67307384 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/db4986aceee84297aee5af1f67307384 2024-12-09T20:44:01,543 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/db4986aceee84297aee5af1f67307384, entries=150, sequenceid=320, filesize=12.0 K 2024-12-09T20:44:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/e9a233cb332c414cbc6e1135f6c26818 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/e9a233cb332c414cbc6e1135f6c26818 2024-12-09T20:44:01,547 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/e9a233cb332c414cbc6e1135f6c26818, entries=150, sequenceid=320, filesize=12.0 K 2024-12-09T20:44:01,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/0602ebcdf4e440b1a1db3580806255ab as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/0602ebcdf4e440b1a1db3580806255ab 2024-12-09T20:44:01,551 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/0602ebcdf4e440b1a1db3580806255ab, entries=150, sequenceid=320, filesize=12.0 K 2024-12-09T20:44:01,552 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 879ac9ed3549bd954296156f21c08ad4 in 1241ms, sequenceid=320, compaction requested=false 2024-12-09T20:44:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-09T20:44:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-09T20:44:01,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-09T20:44:01,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7000 sec 2024-12-09T20:44:01,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7080 sec 2024-12-09T20:44:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:01,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:44:01,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:44:01,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:01,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:44:01,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:01,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:44:01,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:01,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/b956b3e266a84626b42ef0b100ab6c1f is 50, key is test_row_0/A:col10/1733777041665/Put/seqid=0 2024-12-09T20:44:01,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742323_1499 (size=14741) 2024-12-09T20:44:01,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777101783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:01,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777101903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T20:44:02,001 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:02,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-09T20:44:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T20:44:02,004 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:02,004 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:02,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:02,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/b956b3e266a84626b42ef0b100ab6c1f 2024-12-09T20:44:02,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/c0e19252894e40f69215f205431a4bd0 is 50, key is test_row_0/B:col10/1733777041665/Put/seqid=0 2024-12-09T20:44:02,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742324_1500 (size=12301) 2024-12-09T20:44:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T20:44:02,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777102121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:02,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T20:44:02,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777102441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:02,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:02,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:02,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/c0e19252894e40f69215f205431a4bd0 2024-12-09T20:44:02,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/3710717ac65041d08e4192dc34ecbe01 is 50, key is test_row_0/C:col10/1733777041665/Put/seqid=0 2024-12-09T20:44:02,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742325_1501 (size=12301) 2024-12-09T20:44:02,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:02,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:02,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T20:44:02,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777102911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:02,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777102911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:02,913 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,913 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:02,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/3710717ac65041d08e4192dc34ecbe01 2024-12-09T20:44:02,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/b956b3e266a84626b42ef0b100ab6c1f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b956b3e266a84626b42ef0b100ab6c1f 2024-12-09T20:44:02,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:02,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b956b3e266a84626b42ef0b100ab6c1f, entries=200, sequenceid=336, filesize=14.4 K 2024-12-09T20:44:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/c0e19252894e40f69215f205431a4bd0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c0e19252894e40f69215f205431a4bd0 2024-12-09T20:44:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:02,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777102922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:02,922 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:02,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c0e19252894e40f69215f205431a4bd0, entries=150, sequenceid=336, filesize=12.0 K 2024-12-09T20:44:02,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/3710717ac65041d08e4192dc34ecbe01 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3710717ac65041d08e4192dc34ecbe01 2024-12-09T20:44:02,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3710717ac65041d08e4192dc34ecbe01, entries=150, sequenceid=336, filesize=12.0 K 2024-12-09T20:44:02,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 879ac9ed3549bd954296156f21c08ad4 in 1262ms, sequenceid=336, compaction requested=true 2024-12-09T20:44:02,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:02,928 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:02,929 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:02,929 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/A is initiating minor compaction (all files) 2024-12-09T20:44:02,929 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/A in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,929 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/1578037193064f23a6f25fd1ad2a1bcf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/db4986aceee84297aee5af1f67307384, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b956b3e266a84626b42ef0b100ab6c1f] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=39.1 K 2024-12-09T20:44:02,929 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1578037193064f23a6f25fd1ad2a1bcf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733777037566 2024-12-09T20:44:02,930 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting db4986aceee84297aee5af1f67307384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733777039812 2024-12-09T20:44:02,930 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting b956b3e266a84626b42ef0b100ab6c1f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733777040482 2024-12-09T20:44:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:44:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:02,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:44:02,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:02,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:44:02,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:02,933 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:02,933 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:02,933 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/B is initiating minor compaction (all files) 2024-12-09T20:44:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:02,933 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/B in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:02,934 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/7c1c729ef340499793795b8bdc63145a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/e9a233cb332c414cbc6e1135f6c26818, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c0e19252894e40f69215f205431a4bd0] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=36.7 K 2024-12-09T20:44:02,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T20:44:02,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:44:02,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:02,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:44:02,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:02,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:44:02,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:02,934 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c1c729ef340499793795b8bdc63145a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733777037566 2024-12-09T20:44:02,935 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a233cb332c414cbc6e1135f6c26818, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733777039812 2024-12-09T20:44:02,936 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c0e19252894e40f69215f205431a4bd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733777040500 2024-12-09T20:44:02,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/bb016aa691804aac9a62116143e70c53 is 50, key is test_row_0/A:col10/1733777041750/Put/seqid=0 2024-12-09T20:44:02,943 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#A#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:02,943 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/4642e0109ab6442cb0e9bc9263385bab is 50, key is test_row_0/A:col10/1733777041665/Put/seqid=0 2024-12-09T20:44:02,946 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#B#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:02,946 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/923dfac614ae4b9c8b1418f824044c7a is 50, key is test_row_0/B:col10/1733777041665/Put/seqid=0 2024-12-09T20:44:02,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742326_1502 (size=14741) 2024-12-09T20:44:02,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742327_1503 (size=13085) 2024-12-09T20:44:02,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742328_1504 (size=13085) 2024-12-09T20:44:03,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777102989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:03,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777103001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:03,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:03,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777103111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777103111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T20:44:03,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777103321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:03,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777103321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:03,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/bb016aa691804aac9a62116143e70c53 2024-12-09T20:44:03,358 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/4642e0109ab6442cb0e9bc9263385bab as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4642e0109ab6442cb0e9bc9263385bab 2024-12-09T20:44:03,358 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/923dfac614ae4b9c8b1418f824044c7a as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/923dfac614ae4b9c8b1418f824044c7a 2024-12-09T20:44:03,367 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/A of 879ac9ed3549bd954296156f21c08ad4 into 4642e0109ab6442cb0e9bc9263385bab(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:03,367 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:03,367 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/A, priority=13, startTime=1733777042928; duration=0sec 2024-12-09T20:44:03,367 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:03,367 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:A 2024-12-09T20:44:03,367 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:03,367 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/B of 879ac9ed3549bd954296156f21c08ad4 into 923dfac614ae4b9c8b1418f824044c7a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:03,368 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:03,368 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/B, priority=13, startTime=1733777042932; duration=0sec 2024-12-09T20:44:03,368 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:03,368 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:B 2024-12-09T20:44:03,368 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:03,368 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/C is initiating minor compaction (all files) 2024-12-09T20:44:03,368 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/C in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,368 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f1b61dc71013490e9733b80082d3c046, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/0602ebcdf4e440b1a1db3580806255ab, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3710717ac65041d08e4192dc34ecbe01] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=36.7 K 2024-12-09T20:44:03,369 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b61dc71013490e9733b80082d3c046, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733777037566 2024-12-09T20:44:03,369 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0602ebcdf4e440b1a1db3580806255ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733777039812 2024-12-09T20:44:03,369 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3710717ac65041d08e4192dc34ecbe01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733777040500 2024-12-09T20:44:03,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/1a83d11112bc40a3866db03e94972a15 is 50, key is test_row_0/B:col10/1733777041750/Put/seqid=0 2024-12-09T20:44:03,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,399 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:03,400 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/984a2c9b58354e8da53a5109c7263509 is 50, key is test_row_0/C:col10/1733777041665/Put/seqid=0 2024-12-09T20:44:03,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742329_1505 (size=12301) 2024-12-09T20:44:03,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742330_1506 (size=13085) 2024-12-09T20:44:03,435 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/984a2c9b58354e8da53a5109c7263509 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/984a2c9b58354e8da53a5109c7263509 2024-12-09T20:44:03,440 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/C of 879ac9ed3549bd954296156f21c08ad4 into 984a2c9b58354e8da53a5109c7263509(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:03,440 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:03,440 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/C, priority=13, startTime=1733777042933; duration=0sec 2024-12-09T20:44:03,440 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:03,440 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:C 2024-12-09T20:44:03,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777103631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777103631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:03,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/1a83d11112bc40a3866db03e94972a15 2024-12-09T20:44:03,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/821ff7fa8cda4a3a887f37b43ee06de5 is 50, key is test_row_0/C:col10/1733777041750/Put/seqid=0 2024-12-09T20:44:03,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:03,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742331_1507 (size=12301) 2024-12-09T20:44:03,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/821ff7fa8cda4a3a887f37b43ee06de5 2024-12-09T20:44:03,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/bb016aa691804aac9a62116143e70c53 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bb016aa691804aac9a62116143e70c53 2024-12-09T20:44:03,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bb016aa691804aac9a62116143e70c53, entries=200, sequenceid=356, filesize=14.4 K 2024-12-09T20:44:03,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/1a83d11112bc40a3866db03e94972a15 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/1a83d11112bc40a3866db03e94972a15 2024-12-09T20:44:03,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/1a83d11112bc40a3866db03e94972a15, entries=150, sequenceid=356, filesize=12.0 K 2024-12-09T20:44:03,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/821ff7fa8cda4a3a887f37b43ee06de5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/821ff7fa8cda4a3a887f37b43ee06de5 2024-12-09T20:44:03,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/821ff7fa8cda4a3a887f37b43ee06de5, entries=150, sequenceid=356, filesize=12.0 K 2024-12-09T20:44:03,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 879ac9ed3549bd954296156f21c08ad4 in 950ms, sequenceid=356, compaction requested=false 2024-12-09T20:44:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:03,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T20:44:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:03,997 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:44:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:44:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:44:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:03,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:44:03,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:04,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/b2e14e12b14a4576ada08542c2b857bb is 50, key is test_row_0/A:col10/1733777042957/Put/seqid=0 2024-12-09T20:44:04,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742332_1508 (size=12301) 2024-12-09T20:44:04,037 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/b2e14e12b14a4576ada08542c2b857bb 2024-12-09T20:44:04,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/8322b13a9404452fa58b522a457f589d is 50, key is test_row_0/B:col10/1733777042957/Put/seqid=0 2024-12-09T20:44:04,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742333_1509 (size=12301) 2024-12-09T20:44:04,114 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T20:44:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T20:44:04,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:04,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777104202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777104204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:04,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777104321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777104321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:04,509 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/8322b13a9404452fa58b522a457f589d 2024-12-09T20:44:04,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/78ac91f00f914ab8aa8a1376af19a40e is 50, key is test_row_0/C:col10/1733777042957/Put/seqid=0 2024-12-09T20:44:04,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777104542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:04,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777104542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:04,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742334_1510 (size=12301) 2024-12-09T20:44:04,568 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/78ac91f00f914ab8aa8a1376af19a40e 2024-12-09T20:44:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/b2e14e12b14a4576ada08542c2b857bb as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b2e14e12b14a4576ada08542c2b857bb 2024-12-09T20:44:04,580 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b2e14e12b14a4576ada08542c2b857bb, entries=150, sequenceid=371, filesize=12.0 K 2024-12-09T20:44:04,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/8322b13a9404452fa58b522a457f589d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8322b13a9404452fa58b522a457f589d 2024-12-09T20:44:04,588 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8322b13a9404452fa58b522a457f589d, entries=150, sequenceid=371, filesize=12.0 K 2024-12-09T20:44:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/78ac91f00f914ab8aa8a1376af19a40e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/78ac91f00f914ab8aa8a1376af19a40e 2024-12-09T20:44:04,593 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/78ac91f00f914ab8aa8a1376af19a40e, entries=150, sequenceid=371, filesize=12.0 K 2024-12-09T20:44:04,594 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 879ac9ed3549bd954296156f21c08ad4 in 597ms, sequenceid=371, compaction requested=true 2024-12-09T20:44:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-09T20:44:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-09T20:44:04,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-09T20:44:04,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5940 sec 2024-12-09T20:44:04,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.6150 sec 2024-12-09T20:44:04,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:04,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777044197, max timestamp=1733777044869Num uniques 859; 2024-12-09T20:44:04,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777044197, max timestamp=1733777044869Num uniques 859; 2024-12-09T20:44:04,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:44:04,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:44:04,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777044197, max timestamp=1733777044869Num uniques 859; 2024-12-09T20:44:04,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:44:04,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-12-09T20:44:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:44:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:44:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:44:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:04,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/a1006d49da8b45ea856753eefd3992c7 is 50, key is test_row_0/A:col10/1733777044869/Put/seqid=0 2024-12-09T20:44:04,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742335_1511 (size=17181) 2024-12-09T20:44:04,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733777044952, max timestamp=1733777044966Num uniques 859; 2024-12-09T20:44:04,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:44:04,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733777044952, max timestamp=1733777044966Num uniques 859; 2024-12-09T20:44:04,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:44:04,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733777044952, max timestamp=1733777044966Num uniques 859; 2024-12-09T20:44:04,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T20:44:05,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733777045015, max timestamp=1733777045040Num uniques 1068; 2024-12-09T20:44:05,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T20:44:05,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733777045015, max timestamp=1733777045040Num uniques 1068; 2024-12-09T20:44:05,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T20:44:05,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733777045015, max timestamp=1733777045040Num uniques 1068; 2024-12-09T20:44:05,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T20:44:05,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733777045122, max timestamp=1733777045147Num uniques 1077; 2024-12-09T20:44:05,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733777045122, max timestamp=1733777045147Num uniques 1077; 2024-12-09T20:44:05,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-09T20:44:05,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-09T20:44:05,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733777045122, max timestamp=1733777045147Num uniques 1077; 2024-12-09T20:44:05,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-09T20:44:05,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733777045228, max timestamp=1733777045245Num uniques 1086; 2024-12-09T20:44:05,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T20:44:05,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733777045228, max timestamp=1733777045245Num uniques 1086; 2024-12-09T20:44:05,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733777045228, max timestamp=1733777045245Num uniques 1086; 2024-12-09T20:44:05,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T20:44:05,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T20:44:05,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/a1006d49da8b45ea856753eefd3992c7 2024-12-09T20:44:05,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/cb8604c90507410f85d69653864f3335 is 50, key is test_row_0/B:col10/1733777044869/Put/seqid=0 2024-12-09T20:44:05,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742336_1512 (size=12301) 2024-12-09T20:44:05,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733777045300, max timestamp=1733777045329Num uniques 1095; 2024-12-09T20:44:05,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-09T20:44:05,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733777045300, max timestamp=1733777045329Num uniques 1095; 2024-12-09T20:44:05,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-09T20:44:05,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733777045300, max timestamp=1733777045329Num uniques 1095; 2024-12-09T20:44:05,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-09T20:44:05,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733777045358, max timestamp=1733777045423Num uniques 1104; 2024-12-09T20:44:05,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-09T20:44:05,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733777045358, max timestamp=1733777045423Num uniques 1104; 2024-12-09T20:44:05,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-09T20:44:05,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733777045358, max timestamp=1733777045423Num uniques 1104; 2024-12-09T20:44:05,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-09T20:44:05,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733777045464, max timestamp=1733777045508Num uniques 1113; 2024-12-09T20:44:05,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733777045464, max timestamp=1733777045508Num uniques 1113; 2024-12-09T20:44:05,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-09T20:44:05,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-09T20:44:05,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733777045464, max timestamp=1733777045508Num uniques 1113; 2024-12-09T20:44:05,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-09T20:44:05,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733777045625, max timestamp=1733777045648Num uniques 1072; 2024-12-09T20:44:05,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733777045625, max timestamp=1733777045648Num uniques 1072; 2024-12-09T20:44:05,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T20:44:05,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T20:44:05,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733777045625, max timestamp=1733777045648Num uniques 1072; 2024-12-09T20:44:05,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T20:44:05,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/cb8604c90507410f85d69653864f3335 2024-12-09T20:44:05,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/bda7dfb93b0a4c7fb9cb021863cf8ea0 is 50, key is test_row_0/C:col10/1733777044869/Put/seqid=0 2024-12-09T20:44:05,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742337_1513 (size=12301) 2024-12-09T20:44:05,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/bda7dfb93b0a4c7fb9cb021863cf8ea0 2024-12-09T20:44:05,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733777045724, max timestamp=1733777045755Num uniques 1031; 2024-12-09T20:44:05,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T20:44:05,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733777045724, max timestamp=1733777045755Num uniques 1031; 2024-12-09T20:44:05,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T20:44:05,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733777045724, max timestamp=1733777045755Num uniques 1031; 2024-12-09T20:44:05,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T20:44:05,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/a1006d49da8b45ea856753eefd3992c7 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/a1006d49da8b45ea856753eefd3992c7 2024-12-09T20:44:05,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/a1006d49da8b45ea856753eefd3992c7, entries=250, sequenceid=393, filesize=16.8 K 2024-12-09T20:44:05,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/cb8604c90507410f85d69653864f3335 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/cb8604c90507410f85d69653864f3335 2024-12-09T20:44:05,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/cb8604c90507410f85d69653864f3335, entries=150, sequenceid=393, filesize=12.0 K 2024-12-09T20:44:05,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/bda7dfb93b0a4c7fb9cb021863cf8ea0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bda7dfb93b0a4c7fb9cb021863cf8ea0 2024-12-09T20:44:05,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bda7dfb93b0a4c7fb9cb021863cf8ea0, entries=150, sequenceid=393, filesize=12.0 K 2024-12-09T20:44:05,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=46.96 KB/48090 for 879ac9ed3549bd954296156f21c08ad4 in 891ms, sequenceid=393, compaction requested=true 2024-12-09T20:44:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:44:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:05,776 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:44:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:44:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:05,776 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:44:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 879ac9ed3549bd954296156f21c08ad4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:44:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57308 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/B is initiating minor compaction (all files) 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/A is initiating minor compaction (all files) 2024-12-09T20:44:05,777 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/A in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:05,777 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/B in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:05,777 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4642e0109ab6442cb0e9bc9263385bab, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bb016aa691804aac9a62116143e70c53, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b2e14e12b14a4576ada08542c2b857bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/a1006d49da8b45ea856753eefd3992c7] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=56.0 K 2024-12-09T20:44:05,777 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/923dfac614ae4b9c8b1418f824044c7a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/1a83d11112bc40a3866db03e94972a15, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8322b13a9404452fa58b522a457f589d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/cb8604c90507410f85d69653864f3335] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=48.8 K 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4642e0109ab6442cb0e9bc9263385bab, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733777040500 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 923dfac614ae4b9c8b1418f824044c7a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733777040500 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a83d11112bc40a3866db03e94972a15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733777041750 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb016aa691804aac9a62116143e70c53, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733777041750 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8322b13a9404452fa58b522a457f589d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733777042957 2024-12-09T20:44:05,777 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2e14e12b14a4576ada08542c2b857bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733777042957 2024-12-09T20:44:05,778 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1006d49da8b45ea856753eefd3992c7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733777044197 2024-12-09T20:44:05,778 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting cb8604c90507410f85d69653864f3335, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733777044200 2024-12-09T20:44:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:05,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=53.67 KB heapSize=139.17 KB 2024-12-09T20:44:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:44:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:44:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:44:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:05,786 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:05,787 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/13f7b3763ce04fe3bb7bb4cc254af837 is 50, key is test_row_0/B:col10/1733777044869/Put/seqid=0 2024-12-09T20:44:05,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/906df3b89e2544daab77fcc28656b275 is 50, key is test_row_0/A:col10/1733777045778/Put/seqid=0 2024-12-09T20:44:05,807 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#A#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:05,808 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/7c9d213916444a93ae2ba8deefa9e568 is 50, key is test_row_0/A:col10/1733777044869/Put/seqid=0 2024-12-09T20:44:05,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742338_1514 (size=12301) 2024-12-09T20:44:05,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742340_1516 (size=13221) 2024-12-09T20:44:05,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742339_1515 (size=13221) 2024-12-09T20:44:05,843 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/7c9d213916444a93ae2ba8deefa9e568 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/7c9d213916444a93ae2ba8deefa9e568 2024-12-09T20:44:05,848 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/A of 879ac9ed3549bd954296156f21c08ad4 into 7c9d213916444a93ae2ba8deefa9e568(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:05,848 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:05,848 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/A, priority=12, startTime=1733777045776; duration=0sec 2024-12-09T20:44:05,849 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:05,849 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:A 2024-12-09T20:44:05,849 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T20:44:05,849 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T20:44:05,850 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): 879ac9ed3549bd954296156f21c08ad4/C is initiating minor compaction (all files) 2024-12-09T20:44:05,850 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 879ac9ed3549bd954296156f21c08ad4/C in TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:05,850 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/984a2c9b58354e8da53a5109c7263509, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/821ff7fa8cda4a3a887f37b43ee06de5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/78ac91f00f914ab8aa8a1376af19a40e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bda7dfb93b0a4c7fb9cb021863cf8ea0] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp, totalSize=48.8 K 2024-12-09T20:44:05,850 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 984a2c9b58354e8da53a5109c7263509, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733777040500 2024-12-09T20:44:05,850 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 821ff7fa8cda4a3a887f37b43ee06de5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733777041750 2024-12-09T20:44:05,851 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78ac91f00f914ab8aa8a1376af19a40e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733777042957 2024-12-09T20:44:05,851 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting bda7dfb93b0a4c7fb9cb021863cf8ea0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733777044200 2024-12-09T20:44:05,862 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 879ac9ed3549bd954296156f21c08ad4#C#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:05,863 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/5b22b4b8bd1b4ba39696cc8c57def6da is 50, key is test_row_0/C:col10/1733777044869/Put/seqid=0 2024-12-09T20:44:05,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742341_1517 (size=13221) 2024-12-09T20:44:05,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777045857, max timestamp=1733777045886Num uniques 859; 2024-12-09T20:44:05,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777045857, max timestamp=1733777045886Num uniques 859; 2024-12-09T20:44:05,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777045857, max timestamp=1733777045886Num uniques 859; 2024-12-09T20:44:05,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:44:05,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:44:05,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:44:05,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777105980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:05,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777105991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777106101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:06,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777106101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T20:44:06,161 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:06,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-09T20:44:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T20:44:06,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:06,164 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:06,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:06,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/906df3b89e2544daab77fcc28656b275 2024-12-09T20:44:06,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/3cd90164585f460eab92c9aba0ea6174 is 50, key is test_row_0/B:col10/1733777045778/Put/seqid=0 2024-12-09T20:44:06,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742342_1518 (size=12301) 2024-12-09T20:44:06,242 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/13f7b3763ce04fe3bb7bb4cc254af837 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/13f7b3763ce04fe3bb7bb4cc254af837 2024-12-09T20:44:06,259 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/B of 879ac9ed3549bd954296156f21c08ad4 into 13f7b3763ce04fe3bb7bb4cc254af837(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:06,259 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:06,259 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/B, priority=12, startTime=1733777045776; duration=0sec 2024-12-09T20:44:06,259 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:06,259 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:B 2024-12-09T20:44:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T20:44:06,277 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/5b22b4b8bd1b4ba39696cc8c57def6da as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/5b22b4b8bd1b4ba39696cc8c57def6da 2024-12-09T20:44:06,281 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 879ac9ed3549bd954296156f21c08ad4/C of 879ac9ed3549bd954296156f21c08ad4 into 5b22b4b8bd1b4ba39696cc8c57def6da(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:06,281 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:06,281 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., storeName=879ac9ed3549bd954296156f21c08ad4/C, priority=12, startTime=1733777045776; duration=0sec 2024-12-09T20:44:06,282 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:06,282 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 879ac9ed3549bd954296156f21c08ad4:C 2024-12-09T20:44:06,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-09T20:44:06,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:06,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:06,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777106322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777106322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-09T20:44:06,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:06,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T20:44:06,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-09T20:44:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/3cd90164585f460eab92c9aba0ea6174 2024-12-09T20:44:06,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/ff0c4f2cfe9948f9a92be80256b653bf is 50, key is test_row_0/C:col10/1733777045778/Put/seqid=0 2024-12-09T20:44:06,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733777106641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:06,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34400 deadline: 1733777106641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742343_1519 (size=12301) 2024-12-09T20:44:06,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-09T20:44:06,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:06,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T20:44:06,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733777106921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:06,928 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-09T20:44:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:06,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:06,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34368 deadline: 1733777106942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:06,952 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_1' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34388 deadline: 1733777106951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:06,961 INFO [Thread-1961 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1961. 2024-12-09T20:44:06,961 DEBUG [Thread-1961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:06,961 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,961 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:06,962 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_0' in 879ac9ed3549bd954296156f21c08ad4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 , the old value is region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., hostname=00c23a7251f8,36519,1733776895755, seqNum=2 because the exception is null or not the one we care about 2024-12-09T20:44:06,969 INFO [Thread-1963 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1963. 2024-12-09T20:44:06,969 DEBUG [Thread-1963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:06,969 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,970 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:06,970 INFO [Thread-1959 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1959. 2024-12-09T20:44:06,970 DEBUG [Thread-1959 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:06,970 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,970 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:06,974 INFO [Thread-1965 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1965. 2024-12-09T20:44:06,974 DEBUG [Thread-1965 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:06,974 DEBUG [Thread-1965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,974 DEBUG [Thread-1965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:06,983 INFO [Thread-1957 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1957. 2024-12-09T20:44:06,983 DEBUG [Thread-1957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:06,983 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,984 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:06,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:07,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/ff0c4f2cfe9948f9a92be80256b653bf 2024-12-09T20:44:07,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/906df3b89e2544daab77fcc28656b275 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/906df3b89e2544daab77fcc28656b275 2024-12-09T20:44:07,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/906df3b89e2544daab77fcc28656b275, entries=150, sequenceid=554, filesize=12.0 K 2024-12-09T20:44:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/3cd90164585f460eab92c9aba0ea6174 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/3cd90164585f460eab92c9aba0ea6174 2024-12-09T20:44:07,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/3cd90164585f460eab92c9aba0ea6174, entries=150, sequenceid=554, filesize=12.0 K 2024-12-09T20:44:07,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/ff0c4f2cfe9948f9a92be80256b653bf as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/ff0c4f2cfe9948f9a92be80256b653bf 2024-12-09T20:44:07,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/ff0c4f2cfe9948f9a92be80256b653bf, entries=150, sequenceid=554, filesize=12.0 K 2024-12-09T20:44:07,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~139.13 KB/142464, currentSize=134.18 KB/137400 for 879ac9ed3549bd954296156f21c08ad4 in 1294ms, sequenceid=554, compaction requested=false 2024-12-09T20:44:07,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:07,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-09T20:44:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:07,088 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T20:44:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:44:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:44:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:44:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:07,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/cbfc66fef2a54e5d9ab5d94f7942ee3c is 50, key is test_row_0/A:col10/1733777045967/Put/seqid=0 2024-12-09T20:44:07,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742344_1520 (size=12301) 2024-12-09T20:44:07,102 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/cbfc66fef2a54e5d9ab5d94f7942ee3c 2024-12-09T20:44:07,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/774f96485e70451e813c5b4c00515468 is 50, key is test_row_0/B:col10/1733777045967/Put/seqid=0 2024-12-09T20:44:07,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742345_1521 (size=12301) 2024-12-09T20:44:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:07,168 INFO [Thread-1954 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1954. 2024-12-09T20:44:07,168 DEBUG [Thread-1954 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:07,168 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:07,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. as already flushing 2024-12-09T20:44:07,168 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:07,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:07,169 INFO [Thread-1950 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1950. 2024-12-09T20:44:07,169 DEBUG [Thread-1950 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:07,169 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:07,169 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:07,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T20:44:07,515 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/774f96485e70451e813c5b4c00515468 2024-12-09T20:44:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/c89d88d7683f4692a41524039ba43c04 is 50, key is test_row_0/C:col10/1733777045967/Put/seqid=0 2024-12-09T20:44:07,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742346_1522 (size=12301) 2024-12-09T20:44:07,926 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/c89d88d7683f4692a41524039ba43c04 2024-12-09T20:44:07,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/cbfc66fef2a54e5d9ab5d94f7942ee3c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/cbfc66fef2a54e5d9ab5d94f7942ee3c 2024-12-09T20:44:07,934 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/cbfc66fef2a54e5d9ab5d94f7942ee3c, entries=150, sequenceid=594, filesize=12.0 K 2024-12-09T20:44:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/774f96485e70451e813c5b4c00515468 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/774f96485e70451e813c5b4c00515468 2024-12-09T20:44:07,938 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/774f96485e70451e813c5b4c00515468, entries=150, sequenceid=594, filesize=12.0 K 2024-12-09T20:44:07,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/c89d88d7683f4692a41524039ba43c04 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/c89d88d7683f4692a41524039ba43c04 2024-12-09T20:44:07,942 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/c89d88d7683f4692a41524039ba43c04, entries=150, sequenceid=594, filesize=12.0 K 2024-12-09T20:44:07,943 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=13.42 KB/13740 for 879ac9ed3549bd954296156f21c08ad4 in 855ms, sequenceid=594, compaction requested=true 2024-12-09T20:44:07,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 879ac9ed3549bd954296156f21c08ad4: 2024-12-09T20:44:07,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:07,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-09T20:44:07,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-09T20:44:07,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-09T20:44:07,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-12-09T20:44:07,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.7850 sec 2024-12-09T20:44:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T20:44:08,312 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:16,983 INFO [Thread-1948 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1948. 2024-12-09T20:44:16,983 DEBUG [Thread-1948 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:16,983 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:16,983 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:16,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:16,992 INFO [Thread-1946 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1946. 2024-12-09T20:44:16,992 DEBUG [Thread-1946 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:16,992 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:16,992 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:16,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:17,002 INFO [Thread-1952 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1952. 2024-12-09T20:44:17,002 DEBUG [Thread-1952 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:17,002 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:17,002 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:17,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 178 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1510 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4527 rows 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1511 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4529 rows 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1515 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4545 rows 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1517 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4545 rows 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1520 2024-12-09T20:44:17,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4560 rows 2024-12-09T20:44:17,003 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:44:17,003 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:44:17,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:17,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:17,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:17,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T20:44:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T20:44:17,007 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777057007"}]},"ts":"1733777057007"} 2024-12-09T20:44:17,008 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T20:44:17,008 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T20:44:17,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T20:44:17,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=879ac9ed3549bd954296156f21c08ad4, UNASSIGN}] 2024-12-09T20:44:17,010 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=879ac9ed3549bd954296156f21c08ad4, UNASSIGN 2024-12-09T20:44:17,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=879ac9ed3549bd954296156f21c08ad4, regionState=CLOSING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:17,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=879ac9ed3549bd954296156f21c08ad4, UNASSIGN because future has completed 2024-12-09T20:44:17,013 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T20:44:17,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:44:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T20:44:17,165 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:17,165 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 879ac9ed3549bd954296156f21c08ad4, disabling compactions & flushes 2024-12-09T20:44:17,166 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. after waiting 0 ms 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:17,166 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 879ac9ed3549bd954296156f21c08ad4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=A 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=B 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 879ac9ed3549bd954296156f21c08ad4, store=C 2024-12-09T20:44:17,166 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:17,169 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d5717c797337493c8c3aa32f44817ebd is 50, key is test_row_0/A:col10/1733777056991/Put/seqid=0 2024-12-09T20:44:17,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742347_1523 (size=12301) 2024-12-09T20:44:17,173 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d5717c797337493c8c3aa32f44817ebd 2024-12-09T20:44:17,179 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/46395b162dbc4bcdae4bf33f5d1c29be is 50, key is test_row_0/B:col10/1733777056991/Put/seqid=0 2024-12-09T20:44:17,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742348_1524 (size=12301) 2024-12-09T20:44:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T20:44:17,582 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/46395b162dbc4bcdae4bf33f5d1c29be 2024-12-09T20:44:17,588 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/ba45dba8f2124000a7461cab3cb6b66d is 50, key is test_row_0/C:col10/1733777056991/Put/seqid=0 2024-12-09T20:44:17,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742349_1525 (size=12301) 2024-12-09T20:44:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T20:44:17,992 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/ba45dba8f2124000a7461cab3cb6b66d 2024-12-09T20:44:17,996 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/A/d5717c797337493c8c3aa32f44817ebd as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d5717c797337493c8c3aa32f44817ebd 2024-12-09T20:44:18,000 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d5717c797337493c8c3aa32f44817ebd, entries=150, sequenceid=602, filesize=12.0 K 2024-12-09T20:44:18,000 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/B/46395b162dbc4bcdae4bf33f5d1c29be as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/46395b162dbc4bcdae4bf33f5d1c29be 2024-12-09T20:44:18,004 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/46395b162dbc4bcdae4bf33f5d1c29be, entries=150, sequenceid=602, filesize=12.0 K 2024-12-09T20:44:18,004 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/.tmp/C/ba45dba8f2124000a7461cab3cb6b66d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/ba45dba8f2124000a7461cab3cb6b66d 2024-12-09T20:44:18,008 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/ba45dba8f2124000a7461cab3cb6b66d, entries=150, sequenceid=602, filesize=12.0 K 2024-12-09T20:44:18,009 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 879ac9ed3549bd954296156f21c08ad4 in 842ms, sequenceid=602, compaction requested=true 2024-12-09T20:44:18,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/e598b860d9054457b86f3bfbf2ad98ff, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/7ac343aa9389497fba2d527f6b39b717, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bcfad107158441359a3d2efcf04ed017, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d0a09c1e19c44c3592b82f850fa51d24, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dc85c47db73b402f9f9de461445c414f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/051c1e27463f4c6ea0405a710a8d5512, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/277f22b0cda4428ab897ba4b5b82aaba, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dd04056ebd0c4c7bb77f05818fbc611f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/727b0c9954f34a24bd6889ce18b3ca52, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d9bf2d43bd734123859eda77f9d4d840, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/29d3ed84cd9343b2a69de74dd5049265, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/5b0514dd5e114db9824d902e4926c83c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d1b72afea42d4a078a9d394e05b2e1d6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/42146be482ab454eb0d30494415f9784, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4a7ec5331ece43a2b23d1ad58937ba57, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/0b28e8aed1784a71925a1a1470d1016e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d90b77466a504e97a5166ebebba573e5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/f08cf2b364394495bb1576f33e41459a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/661216d5a1954754a8e14474aca3f598, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/1578037193064f23a6f25fd1ad2a1bcf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/834a8b373f9f428aad1826e95ae57716, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/db4986aceee84297aee5af1f67307384, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b956b3e266a84626b42ef0b100ab6c1f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4642e0109ab6442cb0e9bc9263385bab, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bb016aa691804aac9a62116143e70c53, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b2e14e12b14a4576ada08542c2b857bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/a1006d49da8b45ea856753eefd3992c7] to archive 2024-12-09T20:44:18,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:44:18,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/e598b860d9054457b86f3bfbf2ad98ff to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/e598b860d9054457b86f3bfbf2ad98ff 2024-12-09T20:44:18,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/7ac343aa9389497fba2d527f6b39b717 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/7ac343aa9389497fba2d527f6b39b717 2024-12-09T20:44:18,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bcfad107158441359a3d2efcf04ed017 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bcfad107158441359a3d2efcf04ed017 2024-12-09T20:44:18,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d0a09c1e19c44c3592b82f850fa51d24 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d0a09c1e19c44c3592b82f850fa51d24 2024-12-09T20:44:18,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dc85c47db73b402f9f9de461445c414f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dc85c47db73b402f9f9de461445c414f 2024-12-09T20:44:18,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/051c1e27463f4c6ea0405a710a8d5512 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/051c1e27463f4c6ea0405a710a8d5512 2024-12-09T20:44:18,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/277f22b0cda4428ab897ba4b5b82aaba to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/277f22b0cda4428ab897ba4b5b82aaba 2024-12-09T20:44:18,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dd04056ebd0c4c7bb77f05818fbc611f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/dd04056ebd0c4c7bb77f05818fbc611f 2024-12-09T20:44:18,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/727b0c9954f34a24bd6889ce18b3ca52 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/727b0c9954f34a24bd6889ce18b3ca52 2024-12-09T20:44:18,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d9bf2d43bd734123859eda77f9d4d840 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d9bf2d43bd734123859eda77f9d4d840 2024-12-09T20:44:18,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/29d3ed84cd9343b2a69de74dd5049265 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/29d3ed84cd9343b2a69de74dd5049265 2024-12-09T20:44:18,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/5b0514dd5e114db9824d902e4926c83c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/5b0514dd5e114db9824d902e4926c83c 2024-12-09T20:44:18,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d1b72afea42d4a078a9d394e05b2e1d6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d1b72afea42d4a078a9d394e05b2e1d6 2024-12-09T20:44:18,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/42146be482ab454eb0d30494415f9784 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/42146be482ab454eb0d30494415f9784 2024-12-09T20:44:18,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4a7ec5331ece43a2b23d1ad58937ba57 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4a7ec5331ece43a2b23d1ad58937ba57 2024-12-09T20:44:18,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/0b28e8aed1784a71925a1a1470d1016e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/0b28e8aed1784a71925a1a1470d1016e 2024-12-09T20:44:18,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d90b77466a504e97a5166ebebba573e5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d90b77466a504e97a5166ebebba573e5 2024-12-09T20:44:18,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/f08cf2b364394495bb1576f33e41459a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/f08cf2b364394495bb1576f33e41459a 2024-12-09T20:44:18,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/661216d5a1954754a8e14474aca3f598 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/661216d5a1954754a8e14474aca3f598 2024-12-09T20:44:18,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/1578037193064f23a6f25fd1ad2a1bcf to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/1578037193064f23a6f25fd1ad2a1bcf 2024-12-09T20:44:18,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/834a8b373f9f428aad1826e95ae57716 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/834a8b373f9f428aad1826e95ae57716 2024-12-09T20:44:18,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/db4986aceee84297aee5af1f67307384 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/db4986aceee84297aee5af1f67307384 2024-12-09T20:44:18,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b956b3e266a84626b42ef0b100ab6c1f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b956b3e266a84626b42ef0b100ab6c1f 2024-12-09T20:44:18,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4642e0109ab6442cb0e9bc9263385bab to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/4642e0109ab6442cb0e9bc9263385bab 2024-12-09T20:44:18,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bb016aa691804aac9a62116143e70c53 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/bb016aa691804aac9a62116143e70c53 2024-12-09T20:44:18,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b2e14e12b14a4576ada08542c2b857bb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/b2e14e12b14a4576ada08542c2b857bb 2024-12-09T20:44:18,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/a1006d49da8b45ea856753eefd3992c7 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/a1006d49da8b45ea856753eefd3992c7 2024-12-09T20:44:18,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/20f7e61568ec4ce1a6ee8bbc61bc3e23, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/23278f6d89ff483d95957f8ad290e370, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/379b569203f1445c9c57d8da5c6503c3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8d888e2b66e64983a0e1972bca629285, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/6ab1afda55884a4e9a7c51ae4cc60c35, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/a9012c2a394449cbbbef9c6a1f8f8aed, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9924f32b2baf4515bc1d6687498e6a5e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ab8594d2935941919567c76201d11d85, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/dceed8f57ba2448ea6c351ec91498219, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c397380489544146ba682c2d7c6f95ed, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/89e86d8548ab4082abfc3ae7a66c0f4c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/f2cd90889a6e41e194d7f3aa136a4a74, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/bef055b308c44375bd0958b944f00927, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/454697af872a45128cdb16c6d373999a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/0c3f0875c4d54e88a97ac48099b4c158, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/fc49d8eec913426ea7d4f70d5e298865, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ad7387fdaeae47ff9125158ba4fee025, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/b40a2dbf3eb844a682ab373c98a090b1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/54ccf6ead9f44dcc9f02c4878ca0a8fa, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/7c1c729ef340499793795b8bdc63145a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9271db0377e343d29d39d6f944cdb689, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/e9a233cb332c414cbc6e1135f6c26818, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/923dfac614ae4b9c8b1418f824044c7a, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c0e19252894e40f69215f205431a4bd0, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/1a83d11112bc40a3866db03e94972a15, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8322b13a9404452fa58b522a457f589d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/cb8604c90507410f85d69653864f3335] to archive 2024-12-09T20:44:18,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:44:18,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/20f7e61568ec4ce1a6ee8bbc61bc3e23 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/20f7e61568ec4ce1a6ee8bbc61bc3e23 2024-12-09T20:44:18,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/23278f6d89ff483d95957f8ad290e370 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/23278f6d89ff483d95957f8ad290e370 2024-12-09T20:44:18,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/379b569203f1445c9c57d8da5c6503c3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/379b569203f1445c9c57d8da5c6503c3 2024-12-09T20:44:18,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8d888e2b66e64983a0e1972bca629285 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8d888e2b66e64983a0e1972bca629285 2024-12-09T20:44:18,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/6ab1afda55884a4e9a7c51ae4cc60c35 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/6ab1afda55884a4e9a7c51ae4cc60c35 2024-12-09T20:44:18,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/a9012c2a394449cbbbef9c6a1f8f8aed to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/a9012c2a394449cbbbef9c6a1f8f8aed 2024-12-09T20:44:18,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9924f32b2baf4515bc1d6687498e6a5e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9924f32b2baf4515bc1d6687498e6a5e 2024-12-09T20:44:18,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ab8594d2935941919567c76201d11d85 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ab8594d2935941919567c76201d11d85 2024-12-09T20:44:18,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/dceed8f57ba2448ea6c351ec91498219 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/dceed8f57ba2448ea6c351ec91498219 2024-12-09T20:44:18,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c397380489544146ba682c2d7c6f95ed to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c397380489544146ba682c2d7c6f95ed 2024-12-09T20:44:18,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/89e86d8548ab4082abfc3ae7a66c0f4c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/89e86d8548ab4082abfc3ae7a66c0f4c 2024-12-09T20:44:18,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/f2cd90889a6e41e194d7f3aa136a4a74 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/f2cd90889a6e41e194d7f3aa136a4a74 2024-12-09T20:44:18,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/bef055b308c44375bd0958b944f00927 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/bef055b308c44375bd0958b944f00927 2024-12-09T20:44:18,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/454697af872a45128cdb16c6d373999a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/454697af872a45128cdb16c6d373999a 2024-12-09T20:44:18,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/0c3f0875c4d54e88a97ac48099b4c158 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/0c3f0875c4d54e88a97ac48099b4c158 2024-12-09T20:44:18,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/fc49d8eec913426ea7d4f70d5e298865 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/fc49d8eec913426ea7d4f70d5e298865 2024-12-09T20:44:18,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ad7387fdaeae47ff9125158ba4fee025 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/ad7387fdaeae47ff9125158ba4fee025 2024-12-09T20:44:18,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/b40a2dbf3eb844a682ab373c98a090b1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/b40a2dbf3eb844a682ab373c98a090b1 2024-12-09T20:44:18,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/54ccf6ead9f44dcc9f02c4878ca0a8fa to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/54ccf6ead9f44dcc9f02c4878ca0a8fa 2024-12-09T20:44:18,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/7c1c729ef340499793795b8bdc63145a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/7c1c729ef340499793795b8bdc63145a 2024-12-09T20:44:18,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9271db0377e343d29d39d6f944cdb689 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/9271db0377e343d29d39d6f944cdb689 2024-12-09T20:44:18,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/e9a233cb332c414cbc6e1135f6c26818 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/e9a233cb332c414cbc6e1135f6c26818 2024-12-09T20:44:18,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/923dfac614ae4b9c8b1418f824044c7a to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/923dfac614ae4b9c8b1418f824044c7a 2024-12-09T20:44:18,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c0e19252894e40f69215f205431a4bd0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/c0e19252894e40f69215f205431a4bd0 2024-12-09T20:44:18,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/1a83d11112bc40a3866db03e94972a15 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/1a83d11112bc40a3866db03e94972a15 2024-12-09T20:44:18,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8322b13a9404452fa58b522a457f589d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/8322b13a9404452fa58b522a457f589d 2024-12-09T20:44:18,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/cb8604c90507410f85d69653864f3335 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/cb8604c90507410f85d69653864f3335 2024-12-09T20:44:18,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7b8e047cab444a68ad7605eb4b28a644, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/eb054c3dce9e42a195de1ce4357ee3e4, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6e73a3d4a66142bd84ad665331006abc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/61709df56b9b4a3697e27e8d1ca281ee, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6a5531f4d6f744ea8ccc74a6e94374bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/97e66addc1d0436d936b3acdd2d063c3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3bea9972ee564c4d963772d62386fd09, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/29bdb486480f4f44be8bffa93333f4ce, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/5dd9dc0548504748befc7a9eff8182c5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/4aeb3813378f4573b603b3a1cae10f90, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/81d1e07f0c5e40eeaa39a2bb346787e2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7501782285314e569e5bb4e51612273f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/342f25e66776499191db7cf5f1000319, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/38ce96d0181745abb8dd909de1816c12, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/fdb4f47dabd744888d299dfac8d8035e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bf5aa2c80d9c4106a112295c9da5ecdb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6d17c3e3213c42ac8651e7735276bca1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f23a9602ad5643f7b4648f561b13ddcc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/582df2bd4efe42c69f19c159086e85bb, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f1b61dc71013490e9733b80082d3c046, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/45eeb0e6c27e44a09b41fabd36218b74, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/0602ebcdf4e440b1a1db3580806255ab, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/984a2c9b58354e8da53a5109c7263509, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3710717ac65041d08e4192dc34ecbe01, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/821ff7fa8cda4a3a887f37b43ee06de5, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/78ac91f00f914ab8aa8a1376af19a40e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bda7dfb93b0a4c7fb9cb021863cf8ea0] to archive 2024-12-09T20:44:18,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:44:18,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7b8e047cab444a68ad7605eb4b28a644 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7b8e047cab444a68ad7605eb4b28a644 2024-12-09T20:44:18,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/eb054c3dce9e42a195de1ce4357ee3e4 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/eb054c3dce9e42a195de1ce4357ee3e4 2024-12-09T20:44:18,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6e73a3d4a66142bd84ad665331006abc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6e73a3d4a66142bd84ad665331006abc 2024-12-09T20:44:18,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/61709df56b9b4a3697e27e8d1ca281ee to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/61709df56b9b4a3697e27e8d1ca281ee 2024-12-09T20:44:18,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6a5531f4d6f744ea8ccc74a6e94374bb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6a5531f4d6f744ea8ccc74a6e94374bb 2024-12-09T20:44:18,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/97e66addc1d0436d936b3acdd2d063c3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/97e66addc1d0436d936b3acdd2d063c3 2024-12-09T20:44:18,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3bea9972ee564c4d963772d62386fd09 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3bea9972ee564c4d963772d62386fd09 2024-12-09T20:44:18,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/29bdb486480f4f44be8bffa93333f4ce to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/29bdb486480f4f44be8bffa93333f4ce 2024-12-09T20:44:18,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/5dd9dc0548504748befc7a9eff8182c5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/5dd9dc0548504748befc7a9eff8182c5 2024-12-09T20:44:18,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/4aeb3813378f4573b603b3a1cae10f90 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/4aeb3813378f4573b603b3a1cae10f90 2024-12-09T20:44:18,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/81d1e07f0c5e40eeaa39a2bb346787e2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/81d1e07f0c5e40eeaa39a2bb346787e2 2024-12-09T20:44:18,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7501782285314e569e5bb4e51612273f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/7501782285314e569e5bb4e51612273f 2024-12-09T20:44:18,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/342f25e66776499191db7cf5f1000319 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/342f25e66776499191db7cf5f1000319 2024-12-09T20:44:18,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/38ce96d0181745abb8dd909de1816c12 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/38ce96d0181745abb8dd909de1816c12 2024-12-09T20:44:18,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/fdb4f47dabd744888d299dfac8d8035e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/fdb4f47dabd744888d299dfac8d8035e 2024-12-09T20:44:18,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bf5aa2c80d9c4106a112295c9da5ecdb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bf5aa2c80d9c4106a112295c9da5ecdb 2024-12-09T20:44:18,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6d17c3e3213c42ac8651e7735276bca1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/6d17c3e3213c42ac8651e7735276bca1 2024-12-09T20:44:18,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f23a9602ad5643f7b4648f561b13ddcc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f23a9602ad5643f7b4648f561b13ddcc 2024-12-09T20:44:18,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/582df2bd4efe42c69f19c159086e85bb to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/582df2bd4efe42c69f19c159086e85bb 2024-12-09T20:44:18,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f1b61dc71013490e9733b80082d3c046 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/f1b61dc71013490e9733b80082d3c046 2024-12-09T20:44:18,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/45eeb0e6c27e44a09b41fabd36218b74 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/45eeb0e6c27e44a09b41fabd36218b74 2024-12-09T20:44:18,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/0602ebcdf4e440b1a1db3580806255ab to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/0602ebcdf4e440b1a1db3580806255ab 2024-12-09T20:44:18,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/984a2c9b58354e8da53a5109c7263509 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/984a2c9b58354e8da53a5109c7263509 2024-12-09T20:44:18,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3710717ac65041d08e4192dc34ecbe01 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/3710717ac65041d08e4192dc34ecbe01 2024-12-09T20:44:18,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/821ff7fa8cda4a3a887f37b43ee06de5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/821ff7fa8cda4a3a887f37b43ee06de5 2024-12-09T20:44:18,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/78ac91f00f914ab8aa8a1376af19a40e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/78ac91f00f914ab8aa8a1376af19a40e 2024-12-09T20:44:18,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bda7dfb93b0a4c7fb9cb021863cf8ea0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/bda7dfb93b0a4c7fb9cb021863cf8ea0 2024-12-09T20:44:18,108 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/recovered.edits/605.seqid, newMaxSeqId=605, maxSeqId=1 2024-12-09T20:44:18,109 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. 2024-12-09T20:44:18,109 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 879ac9ed3549bd954296156f21c08ad4: Waiting for close lock at 1733777057165Running coprocessor pre-close hooks at 1733777057165Disabling compacts and flushes for region at 1733777057165Disabling writes for close at 1733777057166 (+1 ms)Obtaining lock to block concurrent updates at 1733777057166Preparing flush snapshotting stores in 879ac9ed3549bd954296156f21c08ad4 at 1733777057166Finished memstore snapshotting TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733777057166Flushing stores of TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4. at 1733777057167 (+1 ms)Flushing 879ac9ed3549bd954296156f21c08ad4/A: creating writer at 1733777057167Flushing 879ac9ed3549bd954296156f21c08ad4/A: appending metadata at 1733777057169 (+2 ms)Flushing 879ac9ed3549bd954296156f21c08ad4/A: closing flushed file at 1733777057169Flushing 879ac9ed3549bd954296156f21c08ad4/B: creating writer at 1733777057176 (+7 ms)Flushing 879ac9ed3549bd954296156f21c08ad4/B: appending metadata at 1733777057178 (+2 ms)Flushing 879ac9ed3549bd954296156f21c08ad4/B: closing flushed file at 1733777057178Flushing 879ac9ed3549bd954296156f21c08ad4/C: creating writer at 1733777057586 (+408 ms)Flushing 879ac9ed3549bd954296156f21c08ad4/C: appending metadata at 1733777057588 (+2 ms)Flushing 879ac9ed3549bd954296156f21c08ad4/C: closing flushed file at 1733777057588Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58577d73: reopening flushed file at 1733777057995 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ea278ab: reopening flushed file at 1733777058000 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@695bfb47: reopening flushed file at 1733777058004 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 879ac9ed3549bd954296156f21c08ad4 in 842ms, sequenceid=602, compaction requested=true at 1733777058009 (+5 ms)Writing region close event to WAL at 1733777058104 (+95 ms)Running coprocessor post-close hooks at 1733777058108 (+4 ms)Closed at 1733777058108 2024-12-09T20:44:18,111 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:18,112 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=879ac9ed3549bd954296156f21c08ad4, regionState=CLOSED 2024-12-09T20:44:18,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:44:18,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-09T20:44:18,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 879ac9ed3549bd954296156f21c08ad4, server=00c23a7251f8,36519,1733776895755 in 1.1020 sec 2024-12-09T20:44:18,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-09T20:44:18,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=879ac9ed3549bd954296156f21c08ad4, UNASSIGN in 1.1080 sec 2024-12-09T20:44:18,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-09T20:44:18,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1120 sec 2024-12-09T20:44:18,124 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777058124"}]},"ts":"1733777058124"} 2024-12-09T20:44:18,126 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T20:44:18,126 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T20:44:18,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1230 sec 2024-12-09T20:44:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T20:44:18,141 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:18,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T20:44:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:18,143 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-09T20:44:18,145 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:18,147 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:18,149 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/recovered.edits] 2024-12-09T20:44:18,150 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/7c9d213916444a93ae2ba8deefa9e568 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/7c9d213916444a93ae2ba8deefa9e568 2024-12-09T20:44:18,151 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/906df3b89e2544daab77fcc28656b275 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/906df3b89e2544daab77fcc28656b275 2024-12-09T20:44:18,153 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/cbfc66fef2a54e5d9ab5d94f7942ee3c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/cbfc66fef2a54e5d9ab5d94f7942ee3c 2024-12-09T20:44:18,154 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d5717c797337493c8c3aa32f44817ebd to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/A/d5717c797337493c8c3aa32f44817ebd 2024-12-09T20:44:18,156 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/13f7b3763ce04fe3bb7bb4cc254af837 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/13f7b3763ce04fe3bb7bb4cc254af837 2024-12-09T20:44:18,157 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/3cd90164585f460eab92c9aba0ea6174 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/3cd90164585f460eab92c9aba0ea6174 2024-12-09T20:44:18,160 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/46395b162dbc4bcdae4bf33f5d1c29be to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/46395b162dbc4bcdae4bf33f5d1c29be 2024-12-09T20:44:18,174 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/774f96485e70451e813c5b4c00515468 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/B/774f96485e70451e813c5b4c00515468 2024-12-09T20:44:18,177 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/5b22b4b8bd1b4ba39696cc8c57def6da to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/5b22b4b8bd1b4ba39696cc8c57def6da 2024-12-09T20:44:18,178 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/ba45dba8f2124000a7461cab3cb6b66d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/ba45dba8f2124000a7461cab3cb6b66d 2024-12-09T20:44:18,179 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/c89d88d7683f4692a41524039ba43c04 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/c89d88d7683f4692a41524039ba43c04 2024-12-09T20:44:18,181 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/ff0c4f2cfe9948f9a92be80256b653bf to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/C/ff0c4f2cfe9948f9a92be80256b653bf 2024-12-09T20:44:18,187 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/recovered.edits/605.seqid to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4/recovered.edits/605.seqid 2024-12-09T20:44:18,189 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/879ac9ed3549bd954296156f21c08ad4 2024-12-09T20:44:18,189 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T20:44:18,192 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:18,195 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T20:44:18,198 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T20:44:18,199 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:18,199 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T20:44:18,199 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733777058199"}]},"ts":"9223372036854775807"} 2024-12-09T20:44:18,202 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T20:44:18,202 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 879ac9ed3549bd954296156f21c08ad4, NAME => 'TestAcidGuarantees,,1733777024748.879ac9ed3549bd954296156f21c08ad4.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T20:44:18,202 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T20:44:18,202 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733777058202"}]},"ts":"9223372036854775807"} 2024-12-09T20:44:18,204 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T20:44:18,205 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:18,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-12-09T20:44:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-09T20:44:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T20:44:18,252 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:18,281 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 248), OpenFileDescriptor=441 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=536 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=4021 (was 2775) - AvailableMemoryMB LEAK? - 2024-12-09T20:44:18,295 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=536, ProcessCount=11, AvailableMemoryMB=4010 2024-12-09T20:44:18,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T20:44:18,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:44:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:18,300 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:44:18,300 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:18,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-12-09T20:44:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T20:44:18,301 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:44:18,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742350_1526 (size=1061) 2024-12-09T20:44:18,365 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948 2024-12-09T20:44:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T20:44:18,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742351_1527 (size=53) 2024-12-09T20:44:18,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:44:18,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b46d5304354f6bfcad03af835fcf16f1, disabling compactions & flushes 2024-12-09T20:44:18,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:18,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:18,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. after waiting 0 ms 2024-12-09T20:44:18,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:18,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:18,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b46d5304354f6bfcad03af835fcf16f1: Waiting for close lock at 1733777058427Disabling compacts and flushes for region at 1733777058427Disabling writes for close at 1733777058427Writing region close event to WAL at 1733777058427Closed at 1733777058427 2024-12-09T20:44:18,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:44:18,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733777058429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733777058429"}]},"ts":"1733777058429"} 2024-12-09T20:44:18,431 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:44:18,432 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:44:18,432 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777058432"}]},"ts":"1733777058432"} 2024-12-09T20:44:18,434 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T20:44:18,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, ASSIGN}] 2024-12-09T20:44:18,435 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, ASSIGN 2024-12-09T20:44:18,436 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, ASSIGN; state=OFFLINE, location=00c23a7251f8,36519,1733776895755; forceNewPlan=false, retain=false 2024-12-09T20:44:18,587 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=b46d5304354f6bfcad03af835fcf16f1, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:18,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, ASSIGN because future has completed 2024-12-09T20:44:18,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:44:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T20:44:18,758 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:18,758 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:44:18,758 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,759 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:44:18,759 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,759 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,801 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,802 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:44:18,802 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b46d5304354f6bfcad03af835fcf16f1 columnFamilyName A 2024-12-09T20:44:18,802 DEBUG [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:18,802 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(327): Store=b46d5304354f6bfcad03af835fcf16f1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:44:18,802 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,803 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:44:18,803 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b46d5304354f6bfcad03af835fcf16f1 columnFamilyName B 2024-12-09T20:44:18,803 DEBUG [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:18,804 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(327): Store=b46d5304354f6bfcad03af835fcf16f1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:44:18,804 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,804 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:44:18,804 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b46d5304354f6bfcad03af835fcf16f1 columnFamilyName C 2024-12-09T20:44:18,805 DEBUG [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:18,807 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(327): Store=b46d5304354f6bfcad03af835fcf16f1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:44:18,807 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:18,807 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,810 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,811 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,812 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,812 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,813 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:44:18,816 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,877 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:44:18,878 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened b46d5304354f6bfcad03af835fcf16f1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67598181, jitterRate=0.007291391491889954}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:44:18,878 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:18,879 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for b46d5304354f6bfcad03af835fcf16f1: Running coprocessor pre-open hook at 1733777058759Writing region info on filesystem at 1733777058759Initializing all the Stores at 1733777058759Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733777058759Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733777058800 (+41 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733777058800Cleaning up temporary data from old regions at 1733777058812 (+12 ms)Running coprocessor post-open hooks at 1733777058878 (+66 ms)Region opened successfully at 1733777058879 (+1 ms) 2024-12-09T20:44:18,880 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., pid=146, masterSystemTime=1733777058754 2024-12-09T20:44:18,884 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=b46d5304354f6bfcad03af835fcf16f1, regionState=OPEN, openSeqNum=2, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:18,886 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:18,886 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:18,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:44:18,896 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=00c23a7251f8,36519,1733776895755, table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T20:44:18,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-09T20:44:18,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 in 297 msec 2024-12-09T20:44:18,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-09T20:44:18,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, ASSIGN in 466 msec 2024-12-09T20:44:18,903 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:44:18,904 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777058904"}]},"ts":"1733777058904"} 2024-12-09T20:44:18,906 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T20:44:18,909 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:44:18,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 612 msec 2024-12-09T20:44:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T20:44:18,932 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:18,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:18,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:18,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:18,982 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3097199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:18,983 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a89983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:18,984 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:18,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:18,990 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T20:44:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c23a7251f8,42351,1733776895022 2024-12-09T20:44:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@11ab5798 2024-12-09T20:44:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T20:44:18,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T20:44:19,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T20:44:19,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:44:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:19,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742352_1528 (size=1097) 2024-12-09T20:44:19,555 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T20:44:19,555 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T20:44:19,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T20:44:19,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, REOPEN/MOVE}] 2024-12-09T20:44:19,570 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, REOPEN/MOVE 2024-12-09T20:44:19,571 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b46d5304354f6bfcad03af835fcf16f1, regionState=CLOSING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:19,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, REOPEN/MOVE because future has completed 2024-12-09T20:44:19,575 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T20:44:19,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:44:19,730 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:19,730 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T20:44:19,730 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing b46d5304354f6bfcad03af835fcf16f1, disabling compactions & flushes 2024-12-09T20:44:19,730 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:19,730 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:19,730 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. after waiting 0 ms 2024-12-09T20:44:19,730 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:19,754 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T20:44:19,754 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:19,754 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for b46d5304354f6bfcad03af835fcf16f1: Waiting for close lock at 1733777059730Running coprocessor pre-close hooks at 1733777059730Disabling compacts and flushes for region at 1733777059730Disabling writes for close at 1733777059730Writing region close event to WAL at 1733777059751 (+21 ms)Running coprocessor post-close hooks at 1733777059754 (+3 ms)Closed at 1733777059754 2024-12-09T20:44:19,755 WARN [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: b46d5304354f6bfcad03af835fcf16f1 to self. 2024-12-09T20:44:19,767 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:19,768 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b46d5304354f6bfcad03af835fcf16f1, regionState=CLOSED 2024-12-09T20:44:19,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:44:19,822 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, REOPEN/MOVE; state=CLOSED, location=00c23a7251f8,36519,1733776895755; forceNewPlan=false, retain=true 2024-12-09T20:44:19,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-09T20:44:19,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 in 224 msec 2024-12-09T20:44:19,973 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b46d5304354f6bfcad03af835fcf16f1, regionState=OPENING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:19,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, REOPEN/MOVE because future has completed 2024-12-09T20:44:19,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:44:20,242 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:20,242 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:44:20,243 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,243 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:44:20,243 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,243 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,311 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,312 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:44:20,312 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b46d5304354f6bfcad03af835fcf16f1 columnFamilyName A 2024-12-09T20:44:20,314 DEBUG [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:20,315 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(327): Store=b46d5304354f6bfcad03af835fcf16f1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:44:20,316 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,318 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:44:20,318 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b46d5304354f6bfcad03af835fcf16f1 columnFamilyName B 2024-12-09T20:44:20,318 DEBUG [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:20,384 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(327): Store=b46d5304354f6bfcad03af835fcf16f1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:44:20,384 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,385 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T20:44:20,385 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b46d5304354f6bfcad03af835fcf16f1 columnFamilyName C 2024-12-09T20:44:20,385 DEBUG [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:20,385 INFO [StoreOpener-b46d5304354f6bfcad03af835fcf16f1-1 {}] regionserver.HStore(327): Store=b46d5304354f6bfcad03af835fcf16f1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:44:20,386 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:20,386 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,386 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,387 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,389 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,389 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,389 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T20:44:20,391 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,421 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened b46d5304354f6bfcad03af835fcf16f1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63441846, jitterRate=-0.054642826318740845}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T20:44:20,421 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,422 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for b46d5304354f6bfcad03af835fcf16f1: Running coprocessor pre-open hook at 1733777060243Writing region info on filesystem at 1733777060243Initializing all the Stores at 1733777060244 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733777060244Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733777060311 (+67 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733777060311Cleaning up temporary data from old regions at 1733777060389 (+78 ms)Running coprocessor post-open hooks at 1733777060421 (+32 ms)Region opened successfully at 1733777060422 (+1 ms) 2024-12-09T20:44:20,423 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., pid=151, masterSystemTime=1733777060131 2024-12-09T20:44:20,426 DEBUG [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:20,427 INFO [RS_OPEN_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:20,427 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b46d5304354f6bfcad03af835fcf16f1, regionState=OPEN, openSeqNum=5, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:20,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:44:20,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-12-09T20:44:20,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 in 455 msec 2024-12-09T20:44:20,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-09T20:44:20,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, REOPEN/MOVE in 871 msec 2024-12-09T20:44:20,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-09T20:44:20,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 882 msec 2024-12-09T20:44:20,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.4440 sec 2024-12-09T20:44:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-09T20:44:20,454 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:20,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,458 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,459 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1474c941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,461 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6013f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,514 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3d39d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,518 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be3842d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,529 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a81e8a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0084f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,541 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,543 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,545 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,551 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,554 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,556 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,564 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,568 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,585 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce10fdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,604 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4608fd17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,606 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2461f81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,679 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a758f2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,684 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e7653e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9bca83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,726 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@544c9525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,728 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,730 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a064074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a2ffd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c23a7251f8,42351,-1 for getting cluster id 2024-12-09T20:44:20,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:44:20,748 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df358591-f077-4d91-8b32-36791192d775' 2024-12-09T20:44:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:44:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df358591-f077-4d91-8b32-36791192d775" 2024-12-09T20:44:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e0d94e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c23a7251f8,42351,-1] 2024-12-09T20:44:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:44:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:20,753 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:44:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec776ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:44:20,817 DEBUG [Thread-2383 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:20,818 DEBUG [Thread-2364 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:20,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:20,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:20,823 DEBUG [Thread-2377 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:20,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:20,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T20:44:20,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:20,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:20,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:20,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:20,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:20,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:20,823 DEBUG [Thread-2368 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:20,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:20,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:20,893 DEBUG [Thread-2370 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:20,893 DEBUG [Thread-2381 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:20,894 DEBUG [Thread-2381 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:20,894 DEBUG [Thread-2381 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:20,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:20,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:20,968 DEBUG [Thread-2375 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:20,969 DEBUG [Thread-2372 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:20,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:20,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:20,970 DEBUG [Thread-2366 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:20,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:20,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-09T20:44:20,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:20,974 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T20:44:21,024 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:21,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:21,041 DEBUG [Thread-2379 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:44:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c23a7251f8,36519,1733776895755, seqNum=-1] 2024-12-09T20:44:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:44:21,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:44:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5] 2024-12-09T20:44:21,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733777060897, max timestamp=1733777061046Num uniques 859; 2024-12-09T20:44:21,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:44:21,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733777060897, max timestamp=1733777061046Num uniques 859; 2024-12-09T20:44:21,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:44:21,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733777060897, max timestamp=1733777061046Num uniques 859; 2024-12-09T20:44:21,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T20:44:21,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094e832443dd1547f1bfd8839092c0b297_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777060856/Put/seqid=0 2024-12-09T20:44:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T20:44:21,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:21,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742353_1529 (size=12154) 2024-12-09T20:44:21,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777121193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777121195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48658 deadline: 1733777121195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777121260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T20:44:21,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777121261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777121371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777121372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777121373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48658 deadline: 1733777121374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777121401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,595 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T20:44:21,627 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094e832443dd1547f1bfd8839092c0b297_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094e832443dd1547f1bfd8839092c0b297_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:21,627 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/aaad6036ae3342de957e1b6efea83ef8, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:21,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/aaad6036ae3342de957e1b6efea83ef8 is 175, key is test_row_0/A:col10/1733777060856/Put/seqid=0 2024-12-09T20:44:21,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777121614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777121614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777121614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777121614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48658 deadline: 1733777121614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:21,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742354_1530 (size=30955) 2024-12-09T20:44:21,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:21,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:21,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777121949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777121949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777121950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48658 deadline: 1733777121950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777121950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:21,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:21,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:21,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:21,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T20:44:22,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/aaad6036ae3342de957e1b6efea83ef8 2024-12-09T20:44:22,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/2f4526c92fd44b04b9ae39a6b5d4ba32 is 50, key is test_row_0/B:col10/1733777060856/Put/seqid=0 2024-12-09T20:44:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742355_1531 (size=12001) 2024-12-09T20:44:22,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:22,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:22,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777122461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777122461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:22,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777122461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777122462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48658 deadline: 1733777122461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:22,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/2f4526c92fd44b04b9ae39a6b5d4ba32 2024-12-09T20:44:22,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/8163175e0e5b4ec1a959a81265176ac9 is 50, key is test_row_0/C:col10/1733777060856/Put/seqid=0 2024-12-09T20:44:22,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742356_1532 (size=12001) 2024-12-09T20:44:22,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:22,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:22,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:23,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/8163175e0e5b4ec1a959a81265176ac9 2024-12-09T20:44:23,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/aaad6036ae3342de957e1b6efea83ef8 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/aaad6036ae3342de957e1b6efea83ef8 2024-12-09T20:44:23,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/aaad6036ae3342de957e1b6efea83ef8, entries=150, sequenceid=14, filesize=30.2 K 2024-12-09T20:44:23,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/2f4526c92fd44b04b9ae39a6b5d4ba32 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/2f4526c92fd44b04b9ae39a6b5d4ba32 2024-12-09T20:44:23,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/2f4526c92fd44b04b9ae39a6b5d4ba32, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T20:44:23,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/8163175e0e5b4ec1a959a81265176ac9 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/8163175e0e5b4ec1a959a81265176ac9 2024-12-09T20:44:23,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/8163175e0e5b4ec1a959a81265176ac9, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T20:44:23,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b46d5304354f6bfcad03af835fcf16f1 in 2173ms, sequenceid=14, compaction requested=false 2024-12-09T20:44:23,034 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T20:44:23,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:23,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T20:44:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:23,036 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-09T20:44:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:23,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098a45a60b5d4143f3a98f50d0fff2e75e_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777061260/Put/seqid=0 2024-12-09T20:44:23,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742357_1533 (size=12154) 2024-12-09T20:44:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T20:44:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:23,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:23,474 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098a45a60b5d4143f3a98f50d0fff2e75e_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098a45a60b5d4143f3a98f50d0fff2e75e_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/bcb93efd14fa4b2fa36c5d9b4000419c, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:23,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/bcb93efd14fa4b2fa36c5d9b4000419c is 175, key is test_row_0/A:col10/1733777061260/Put/seqid=0 2024-12-09T20:44:23,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742358_1534 (size=30955) 2024-12-09T20:44:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777123478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777123479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777123480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777123481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48658 deadline: 1733777123481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,484 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in b46d5304354f6bfcad03af835fcf16f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777123591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777123591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777123591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777123591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,622 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:44:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777123801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777123801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777123801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777123801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:23,879 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/bcb93efd14fa4b2fa36c5d9b4000419c 2024-12-09T20:44:23,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/4911d148d21449b683b11319802cb4b3 is 50, key is test_row_0/B:col10/1733777061260/Put/seqid=0 2024-12-09T20:44:23,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742359_1535 (size=12001) 2024-12-09T20:44:23,898 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/4911d148d21449b683b11319802cb4b3 2024-12-09T20:44:23,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/d9602334866146d8ad5de2037192fabf is 50, key is test_row_0/C:col10/1733777061260/Put/seqid=0 2024-12-09T20:44:23,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742360_1536 (size=12001) 2024-12-09T20:44:24,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777124111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777124111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777124111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777124111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,314 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/d9602334866146d8ad5de2037192fabf 2024-12-09T20:44:24,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/bcb93efd14fa4b2fa36c5d9b4000419c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/bcb93efd14fa4b2fa36c5d9b4000419c 2024-12-09T20:44:24,333 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/bcb93efd14fa4b2fa36c5d9b4000419c, entries=150, sequenceid=49, filesize=30.2 K 2024-12-09T20:44:24,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/4911d148d21449b683b11319802cb4b3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4911d148d21449b683b11319802cb4b3 2024-12-09T20:44:24,340 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4911d148d21449b683b11319802cb4b3, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T20:44:24,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-12-09T20:44:24,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/d9602334866146d8ad5de2037192fabf as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/d9602334866146d8ad5de2037192fabf 2024-12-09T20:44:24,354 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/d9602334866146d8ad5de2037192fabf, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T20:44:24,355 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for b46d5304354f6bfcad03af835fcf16f1 in 1318ms, sequenceid=49, compaction requested=false 2024-12-09T20:44:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-09T20:44:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-09T20:44:24,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-09T20:44:24,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3380 sec 2024-12-09T20:44:24,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.3930 sec 2024-12-09T20:44:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:24,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T20:44:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:24,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092b1ce865fb784d4e97339e6a374d26bc_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777063479/Put/seqid=0 2024-12-09T20:44:24,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742361_1537 (size=12154) 2024-12-09T20:44:24,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777124643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777124644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777124644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777124646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777124752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777124752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777124752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777124752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777124961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777124961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777124961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777124961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,045 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:25,048 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092b1ce865fb784d4e97339e6a374d26bc_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092b1ce865fb784d4e97339e6a374d26bc_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:25,048 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/dbf0c933a3d24802b1a738d01a56bf57, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:25,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/dbf0c933a3d24802b1a738d01a56bf57 is 175, key is test_row_0/A:col10/1733777063479/Put/seqid=0 2024-12-09T20:44:25,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742362_1538 (size=30955) 2024-12-09T20:44:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T20:44:25,142 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:25,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-09T20:44:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T20:44:25,143 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:25,144 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:25,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T20:44:25,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777125271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777125272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:25,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777125272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777125272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T20:44:25,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:25,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T20:44:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,452 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/dbf0c933a3d24802b1a738d01a56bf57 2024-12-09T20:44:25,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/cf5344b7ddea47fc9e8e96310091efe6 is 50, key is test_row_0/B:col10/1733777063479/Put/seqid=0 2024-12-09T20:44:25,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742363_1539 (size=12001) 2024-12-09T20:44:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T20:44:25,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48658 deadline: 1733777125501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:25,503 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in b46d5304354f6bfcad03af835fcf16f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4308 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T20:44:25,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T20:44:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T20:44:25,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777125781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777125781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777125781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777125781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:25,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/cf5344b7ddea47fc9e8e96310091efe6 2024-12-09T20:44:25,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/90be9b96b9504481b878710d8613f237 is 50, key is test_row_0/C:col10/1733777063479/Put/seqid=0 2024-12-09T20:44:25,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742364_1540 (size=12001) 2024-12-09T20:44:25,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T20:44:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:25,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:26,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T20:44:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:26,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T20:44:26,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:26,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:26,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:26,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/90be9b96b9504481b878710d8613f237 2024-12-09T20:44:26,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/dbf0c933a3d24802b1a738d01a56bf57 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dbf0c933a3d24802b1a738d01a56bf57 2024-12-09T20:44:26,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dbf0c933a3d24802b1a738d01a56bf57, entries=150, sequenceid=62, filesize=30.2 K 2024-12-09T20:44:26,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/cf5344b7ddea47fc9e8e96310091efe6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/cf5344b7ddea47fc9e8e96310091efe6 2024-12-09T20:44:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T20:44:26,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/cf5344b7ddea47fc9e8e96310091efe6, entries=150, sequenceid=62, filesize=11.7 K 2024-12-09T20:44:26,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/90be9b96b9504481b878710d8613f237 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/90be9b96b9504481b878710d8613f237 2024-12-09T20:44:26,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/90be9b96b9504481b878710d8613f237, entries=150, sequenceid=62, filesize=11.7 K 2024-12-09T20:44:26,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b46d5304354f6bfcad03af835fcf16f1 in 1675ms, sequenceid=62, compaction requested=true 2024-12-09T20:44:26,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:26,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:44:26,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:26,299 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:26,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:44:26,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:26,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:44:26,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:26,299 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:26,301 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:26,301 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:26,301 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/B is initiating minor compaction (all files) 2024-12-09T20:44:26,301 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/A is initiating minor compaction (all files) 2024-12-09T20:44:26,301 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/A in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,301 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/aaad6036ae3342de957e1b6efea83ef8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/bcb93efd14fa4b2fa36c5d9b4000419c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dbf0c933a3d24802b1a738d01a56bf57] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=90.7 K 2024-12-09T20:44:26,301 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,302 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/aaad6036ae3342de957e1b6efea83ef8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/bcb93efd14fa4b2fa36c5d9b4000419c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dbf0c933a3d24802b1a738d01a56bf57] 2024-12-09T20:44:26,302 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/B in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,302 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/2f4526c92fd44b04b9ae39a6b5d4ba32, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4911d148d21449b683b11319802cb4b3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/cf5344b7ddea47fc9e8e96310091efe6] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=35.2 K 2024-12-09T20:44:26,302 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4526c92fd44b04b9ae39a6b5d4ba32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733777060853 2024-12-09T20:44:26,302 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaad6036ae3342de957e1b6efea83ef8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733777060853 2024-12-09T20:44:26,303 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 4911d148d21449b683b11319802cb4b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733777061192 2024-12-09T20:44:26,303 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcb93efd14fa4b2fa36c5d9b4000419c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733777061192 2024-12-09T20:44:26,303 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting cf5344b7ddea47fc9e8e96310091efe6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733777063479 2024-12-09T20:44:26,303 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbf0c933a3d24802b1a738d01a56bf57, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733777063479 2024-12-09T20:44:26,312 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#B#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:26,312 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/371deb168daf4412bd6d855dfa24a53b is 50, key is test_row_0/B:col10/1733777063479/Put/seqid=0 2024-12-09T20:44:26,314 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:26,322 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209dada9523fe28456b82f163d5570e24f5_b46d5304354f6bfcad03af835fcf16f1 store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:26,323 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209dada9523fe28456b82f163d5570e24f5_b46d5304354f6bfcad03af835fcf16f1, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:26,324 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209dada9523fe28456b82f163d5570e24f5_b46d5304354f6bfcad03af835fcf16f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:26,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742365_1541 (size=12104) 2024-12-09T20:44:26,345 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/371deb168daf4412bd6d855dfa24a53b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/371deb168daf4412bd6d855dfa24a53b 2024-12-09T20:44:26,350 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/B of b46d5304354f6bfcad03af835fcf16f1 into 371deb168daf4412bd6d855dfa24a53b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:26,350 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:26,350 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/B, priority=13, startTime=1733777066299; duration=0sec 2024-12-09T20:44:26,350 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:26,350 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:B 2024-12-09T20:44:26,350 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:26,351 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:26,351 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/C is initiating minor compaction (all files) 2024-12-09T20:44:26,351 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/C in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,351 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/8163175e0e5b4ec1a959a81265176ac9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/d9602334866146d8ad5de2037192fabf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/90be9b96b9504481b878710d8613f237] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=35.2 K 2024-12-09T20:44:26,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742366_1542 (size=4469) 2024-12-09T20:44:26,352 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 8163175e0e5b4ec1a959a81265176ac9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733777060853 2024-12-09T20:44:26,353 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting d9602334866146d8ad5de2037192fabf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733777061192 2024-12-09T20:44:26,353 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 90be9b96b9504481b878710d8613f237, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733777063479 2024-12-09T20:44:26,359 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#C#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:26,360 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/78bc81f646da416482c9ae47bd971d22 is 50, key is test_row_0/C:col10/1733777063479/Put/seqid=0 2024-12-09T20:44:26,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T20:44:26,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,365 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:44:26,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:26,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:26,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:26,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:26,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:26,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:26,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742367_1543 (size=12104) 2024-12-09T20:44:26,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120930d99b42b1c4440aa6500910edccee3d_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777064637/Put/seqid=0 2024-12-09T20:44:26,384 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/78bc81f646da416482c9ae47bd971d22 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/78bc81f646da416482c9ae47bd971d22 2024-12-09T20:44:26,394 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/C of b46d5304354f6bfcad03af835fcf16f1 into 78bc81f646da416482c9ae47bd971d22(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:26,394 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:26,394 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/C, priority=13, startTime=1733777066299; duration=0sec 2024-12-09T20:44:26,394 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:26,394 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:C 2024-12-09T20:44:26,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742368_1544 (size=12154) 2024-12-09T20:44:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:26,403 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120930d99b42b1c4440aa6500910edccee3d_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120930d99b42b1c4440aa6500910edccee3d_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:26,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/2218caab7e454ab1b6fd1486261a8020, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:26,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/2218caab7e454ab1b6fd1486261a8020 is 175, key is test_row_0/A:col10/1733777064637/Put/seqid=0 2024-12-09T20:44:26,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742369_1545 (size=30955) 2024-12-09T20:44:26,753 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#A#compaction#463 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:26,753 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/183162e9f6b34511b4dcea217a94253b is 175, key is test_row_0/A:col10/1733777063479/Put/seqid=0 2024-12-09T20:44:26,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742370_1546 (size=31058) 2024-12-09T20:44:26,779 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/183162e9f6b34511b4dcea217a94253b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/183162e9f6b34511b4dcea217a94253b 2024-12-09T20:44:26,785 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/A of b46d5304354f6bfcad03af835fcf16f1 into 183162e9f6b34511b4dcea217a94253b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:26,785 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:26,785 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/A, priority=13, startTime=1733777066299; duration=0sec 2024-12-09T20:44:26,785 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:26,785 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:A 2024-12-09T20:44:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:26,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777126806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:26,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777126811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:26,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777126812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777126812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:26,832 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/2218caab7e454ab1b6fd1486261a8020 2024-12-09T20:44:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/3bc537c640714a0c82f93460cfecc695 is 50, key is test_row_0/B:col10/1733777064637/Put/seqid=0 2024-12-09T20:44:26,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742371_1547 (size=12001) 2024-12-09T20:44:26,874 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/3bc537c640714a0c82f93460cfecc695 2024-12-09T20:44:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/7968499f29f24874b9f5ed227141ce07 is 50, key is test_row_0/C:col10/1733777064637/Put/seqid=0 2024-12-09T20:44:26,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742372_1548 (size=12001) 2024-12-09T20:44:26,896 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/7968499f29f24874b9f5ed227141ce07 2024-12-09T20:44:26,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/2218caab7e454ab1b6fd1486261a8020 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2218caab7e454ab1b6fd1486261a8020 2024-12-09T20:44:26,903 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2218caab7e454ab1b6fd1486261a8020, entries=150, sequenceid=82, filesize=30.2 K 2024-12-09T20:44:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/3bc537c640714a0c82f93460cfecc695 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/3bc537c640714a0c82f93460cfecc695 2024-12-09T20:44:26,906 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/3bc537c640714a0c82f93460cfecc695, entries=150, sequenceid=82, filesize=11.7 K 2024-12-09T20:44:26,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/7968499f29f24874b9f5ed227141ce07 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7968499f29f24874b9f5ed227141ce07 2024-12-09T20:44:26,911 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7968499f29f24874b9f5ed227141ce07, entries=150, sequenceid=82, filesize=11.7 K 2024-12-09T20:44:26,911 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for b46d5304354f6bfcad03af835fcf16f1 in 546ms, sequenceid=82, compaction requested=false 2024-12-09T20:44:26,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:26,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:26,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-09T20:44:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-09T20:44:26,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-09T20:44:26,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7680 sec 2024-12-09T20:44:26,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.7730 sec 2024-12-09T20:44:26,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T20:44:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:26,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:26,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:26,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:26,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:26,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:26,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:26,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d01663a71a72481cad5349a3c7785b02_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777066804/Put/seqid=0 2024-12-09T20:44:26,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777126939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:26,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777126939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:26,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777126941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:26,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777126942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:26,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742373_1549 (size=17034) 2024-12-09T20:44:26,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:26,969 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d01663a71a72481cad5349a3c7785b02_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d01663a71a72481cad5349a3c7785b02_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:26,970 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/6534a00b56b6477daa893e403d8a2d51, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:26,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/6534a00b56b6477daa893e403d8a2d51 is 175, key is test_row_0/A:col10/1733777066804/Put/seqid=0 2024-12-09T20:44:26,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742374_1550 (size=48139) 2024-12-09T20:44:27,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777127051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777127051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777127051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777127052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777127261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777127261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777127261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777127261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T20:44:27,301 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:27,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-09T20:44:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T20:44:27,302 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:27,303 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:27,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:27,398 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/6534a00b56b6477daa893e403d8a2d51 2024-12-09T20:44:27,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/acea48f1ad6340919d583c6cefb59549 is 50, key is test_row_0/B:col10/1733777066804/Put/seqid=0 2024-12-09T20:44:27,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742375_1551 (size=12001) 2024-12-09T20:44:27,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/acea48f1ad6340919d583c6cefb59549 2024-12-09T20:44:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T20:44:27,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/a4a9c37d9f414d33b7095902d06283ce is 50, key is test_row_0/C:col10/1733777066804/Put/seqid=0 2024-12-09T20:44:27,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742376_1552 (size=12001) 2024-12-09T20:44:27,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T20:44:27,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:27,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:27,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777127571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777127571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777127571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777127571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:27,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T20:44:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T20:44:27,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T20:44:27,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:27,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:27,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:27,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/a4a9c37d9f414d33b7095902d06283ce 2024-12-09T20:44:27,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/6534a00b56b6477daa893e403d8a2d51 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/6534a00b56b6477daa893e403d8a2d51 2024-12-09T20:44:27,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/6534a00b56b6477daa893e403d8a2d51, entries=250, sequenceid=101, filesize=47.0 K 2024-12-09T20:44:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/acea48f1ad6340919d583c6cefb59549 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/acea48f1ad6340919d583c6cefb59549 2024-12-09T20:44:27,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/acea48f1ad6340919d583c6cefb59549, entries=150, sequenceid=101, filesize=11.7 K 2024-12-09T20:44:27,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/a4a9c37d9f414d33b7095902d06283ce as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/a4a9c37d9f414d33b7095902d06283ce 2024-12-09T20:44:27,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/a4a9c37d9f414d33b7095902d06283ce, entries=150, sequenceid=101, filesize=11.7 K 2024-12-09T20:44:27,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b46d5304354f6bfcad03af835fcf16f1 in 903ms, sequenceid=101, compaction requested=true 2024-12-09T20:44:27,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:27,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:44:27,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:27,826 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:27,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:44:27,826 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:27,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:27,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:44:27,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:27,827 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:27,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/A is initiating minor compaction (all files) 2024-12-09T20:44:27,828 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:27,828 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/A in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,828 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/B is initiating minor compaction (all files) 2024-12-09T20:44:27,828 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/B in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,828 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/183162e9f6b34511b4dcea217a94253b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2218caab7e454ab1b6fd1486261a8020, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/6534a00b56b6477daa893e403d8a2d51] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=107.6 K 2024-12-09T20:44:27,828 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/371deb168daf4412bd6d855dfa24a53b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/3bc537c640714a0c82f93460cfecc695, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/acea48f1ad6340919d583c6cefb59549] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=35.3 K 2024-12-09T20:44:27,828 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/183162e9f6b34511b4dcea217a94253b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2218caab7e454ab1b6fd1486261a8020, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/6534a00b56b6477daa893e403d8a2d51] 2024-12-09T20:44:27,828 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 371deb168daf4412bd6d855dfa24a53b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733777063479 2024-12-09T20:44:27,828 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc537c640714a0c82f93460cfecc695, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733777064637 2024-12-09T20:44:27,828 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 183162e9f6b34511b4dcea217a94253b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733777063479 2024-12-09T20:44:27,828 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting acea48f1ad6340919d583c6cefb59549, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733777066804 2024-12-09T20:44:27,829 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2218caab7e454ab1b6fd1486261a8020, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733777064637 2024-12-09T20:44:27,829 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6534a00b56b6477daa893e403d8a2d51, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733777066804 2024-12-09T20:44:27,836 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#B#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:27,837 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/54c4dcc8e1394315bea8fd4813260d8f is 50, key is test_row_0/B:col10/1733777066804/Put/seqid=0 2024-12-09T20:44:27,838 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:27,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742377_1553 (size=12207) 2024-12-09T20:44:27,840 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120948fbad59d4c544638a189c85a059128c_b46d5304354f6bfcad03af835fcf16f1 store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:27,842 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120948fbad59d4c544638a189c85a059128c_b46d5304354f6bfcad03af835fcf16f1, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:27,842 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120948fbad59d4c544638a189c85a059128c_b46d5304354f6bfcad03af835fcf16f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:27,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742378_1554 (size=4469) 2024-12-09T20:44:27,846 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#A#compaction#472 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:27,847 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/98d0c0c900444a81ac2664ec87ea18ed is 175, key is test_row_0/A:col10/1733777066804/Put/seqid=0 2024-12-09T20:44:27,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742379_1555 (size=31161) 2024-12-09T20:44:27,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T20:44:27,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:27,912 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T20:44:27,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:27,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:27,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:27,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:27,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:27,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095b58f0f73c504917849bb68ce957bf02_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777066937/Put/seqid=0 2024-12-09T20:44:27,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742380_1556 (size=12154) 2024-12-09T20:44:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T20:44:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:28,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:28,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777128094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777128094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777128100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777128100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777128211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777128211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777128211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777128211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,243 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/54c4dcc8e1394315bea8fd4813260d8f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/54c4dcc8e1394315bea8fd4813260d8f 2024-12-09T20:44:28,246 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/B of b46d5304354f6bfcad03af835fcf16f1 into 54c4dcc8e1394315bea8fd4813260d8f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:28,246 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:28,246 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/B, priority=13, startTime=1733777067826; duration=0sec 2024-12-09T20:44:28,246 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:28,246 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:B 2024-12-09T20:44:28,246 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:28,246 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:28,247 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/C is initiating minor compaction (all files) 2024-12-09T20:44:28,247 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/C in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:28,247 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/78bc81f646da416482c9ae47bd971d22, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7968499f29f24874b9f5ed227141ce07, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/a4a9c37d9f414d33b7095902d06283ce] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=35.3 K 2024-12-09T20:44:28,247 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 78bc81f646da416482c9ae47bd971d22, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733777063479 2024-12-09T20:44:28,247 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 7968499f29f24874b9f5ed227141ce07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733777064637 2024-12-09T20:44:28,247 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a9c37d9f414d33b7095902d06283ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733777066804 2024-12-09T20:44:28,252 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:28,252 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/0e90a816693c4fba888104e386e460d1 is 50, key is test_row_0/C:col10/1733777066804/Put/seqid=0 2024-12-09T20:44:28,252 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/98d0c0c900444a81ac2664ec87ea18ed as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/98d0c0c900444a81ac2664ec87ea18ed 2024-12-09T20:44:28,255 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/A of b46d5304354f6bfcad03af835fcf16f1 into 98d0c0c900444a81ac2664ec87ea18ed(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:28,255 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:28,255 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/A, priority=13, startTime=1733777067826; duration=0sec 2024-12-09T20:44:28,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742381_1557 (size=12207) 2024-12-09T20:44:28,255 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:28,256 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:A 2024-12-09T20:44:28,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:28,323 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095b58f0f73c504917849bb68ce957bf02_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095b58f0f73c504917849bb68ce957bf02_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:28,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/67d5fcdda8c04c0ca776c55cd6916884, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:28,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/67d5fcdda8c04c0ca776c55cd6916884 is 175, key is test_row_0/A:col10/1733777066937/Put/seqid=0 2024-12-09T20:44:28,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742382_1558 (size=30955) 2024-12-09T20:44:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777128421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777128421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777128421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777128421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T20:44:28,659 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/0e90a816693c4fba888104e386e460d1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/0e90a816693c4fba888104e386e460d1 2024-12-09T20:44:28,663 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/C of b46d5304354f6bfcad03af835fcf16f1 into 0e90a816693c4fba888104e386e460d1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:28,663 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:28,663 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/C, priority=13, startTime=1733777067826; duration=0sec 2024-12-09T20:44:28,663 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:28,663 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:C 2024-12-09T20:44:28,727 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/67d5fcdda8c04c0ca776c55cd6916884 2024-12-09T20:44:28,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/6cd5400576324f11a68af9dc4b5c16a2 is 50, key is test_row_0/B:col10/1733777066937/Put/seqid=0 2024-12-09T20:44:28,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777128731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777128731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742383_1559 (size=12001) 2024-12-09T20:44:28,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777128741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:28,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777128741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:29,136 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/6cd5400576324f11a68af9dc4b5c16a2 2024-12-09T20:44:29,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/770d6a6634324abbad18e6d60f622dc1 is 50, key is test_row_0/C:col10/1733777066937/Put/seqid=0 2024-12-09T20:44:29,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742384_1560 (size=12001) 2024-12-09T20:44:29,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777129241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:29,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777129241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:29,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:29,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777129251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777129251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T20:44:29,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48658 deadline: 1733777129511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:29,513 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in b46d5304354f6bfcad03af835fcf16f1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8318 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:29,545 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/770d6a6634324abbad18e6d60f622dc1 2024-12-09T20:44:29,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/67d5fcdda8c04c0ca776c55cd6916884 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/67d5fcdda8c04c0ca776c55cd6916884 2024-12-09T20:44:29,560 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/67d5fcdda8c04c0ca776c55cd6916884, entries=150, sequenceid=116, filesize=30.2 K 2024-12-09T20:44:29,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/6cd5400576324f11a68af9dc4b5c16a2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6cd5400576324f11a68af9dc4b5c16a2 2024-12-09T20:44:29,564 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6cd5400576324f11a68af9dc4b5c16a2, entries=150, sequenceid=116, filesize=11.7 K 2024-12-09T20:44:29,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/770d6a6634324abbad18e6d60f622dc1 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/770d6a6634324abbad18e6d60f622dc1 2024-12-09T20:44:29,568 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/770d6a6634324abbad18e6d60f622dc1, entries=150, sequenceid=116, filesize=11.7 K 2024-12-09T20:44:29,568 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b46d5304354f6bfcad03af835fcf16f1 in 1656ms, sequenceid=116, compaction requested=false 2024-12-09T20:44:29,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:29,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:29,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-09T20:44:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-09T20:44:29,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-09T20:44:29,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2670 sec 2024-12-09T20:44:29,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.2700 sec 2024-12-09T20:44:30,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T20:44:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:30,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:30,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:30,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:30,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:30,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:30,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:30,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120981bb73fd5fb74cea9b20f5dddca24f47_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777068098/Put/seqid=0 2024-12-09T20:44:30,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742385_1561 (size=14744) 2024-12-09T20:44:30,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777130313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777130319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777130320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777130320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777130421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777130431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777130431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777130431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777130631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777130641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777130641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777130641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,690 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:30,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120981bb73fd5fb74cea9b20f5dddca24f47_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120981bb73fd5fb74cea9b20f5dddca24f47_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:30,693 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/2ba76249f5684f4ab8c2034c0e77f249, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:30,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/2ba76249f5684f4ab8c2034c0e77f249 is 175, key is test_row_0/A:col10/1733777068098/Put/seqid=0 2024-12-09T20:44:30,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742386_1562 (size=39699) 2024-12-09T20:44:30,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777130941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777130951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777130951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777130951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:31,097 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/2ba76249f5684f4ab8c2034c0e77f249 2024-12-09T20:44:31,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/9d9244ac5bfb4763a3671e3d883e8651 is 50, key is test_row_0/B:col10/1733777068098/Put/seqid=0 2024-12-09T20:44:31,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742387_1563 (size=12151) 2024-12-09T20:44:31,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/9d9244ac5bfb4763a3671e3d883e8651 2024-12-09T20:44:31,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/310ccc319d5f4311ab832aaee231127e is 50, key is test_row_0/C:col10/1733777068098/Put/seqid=0 2024-12-09T20:44:31,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742388_1564 (size=12151) 2024-12-09T20:44:31,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777131451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:31,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777131461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:31,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777131461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:31,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777131461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T20:44:31,471 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:31,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-09T20:44:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T20:44:31,473 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:31,474 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:31,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:31,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/310ccc319d5f4311ab832aaee231127e 2024-12-09T20:44:31,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/2ba76249f5684f4ab8c2034c0e77f249 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2ba76249f5684f4ab8c2034c0e77f249 2024-12-09T20:44:31,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2ba76249f5684f4ab8c2034c0e77f249, entries=200, sequenceid=138, filesize=38.8 K 2024-12-09T20:44:31,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/9d9244ac5bfb4763a3671e3d883e8651 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/9d9244ac5bfb4763a3671e3d883e8651 2024-12-09T20:44:31,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/9d9244ac5bfb4763a3671e3d883e8651, entries=150, sequenceid=138, filesize=11.9 K 2024-12-09T20:44:31,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/310ccc319d5f4311ab832aaee231127e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/310ccc319d5f4311ab832aaee231127e 2024-12-09T20:44:31,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/310ccc319d5f4311ab832aaee231127e, entries=150, sequenceid=138, filesize=11.9 K 2024-12-09T20:44:31,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for b46d5304354f6bfcad03af835fcf16f1 in 1278ms, sequenceid=138, compaction requested=true 2024-12-09T20:44:31,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:31,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:44:31,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:31,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:44:31,531 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:31,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:31,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:44:31,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T20:44:31,532 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:31,532 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:31,532 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/A is initiating minor compaction (all files) 2024-12-09T20:44:31,532 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/A in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:31,532 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/98d0c0c900444a81ac2664ec87ea18ed, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/67d5fcdda8c04c0ca776c55cd6916884, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2ba76249f5684f4ab8c2034c0e77f249] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=99.4 K 2024-12-09T20:44:31,532 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:31,532 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/98d0c0c900444a81ac2664ec87ea18ed, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/67d5fcdda8c04c0ca776c55cd6916884, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2ba76249f5684f4ab8c2034c0e77f249] 2024-12-09T20:44:31,533 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:31,533 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/B is initiating minor compaction (all files) 2024-12-09T20:44:31,533 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 98d0c0c900444a81ac2664ec87ea18ed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733777066804 2024-12-09T20:44:31,533 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/B in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:31,533 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/54c4dcc8e1394315bea8fd4813260d8f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6cd5400576324f11a68af9dc4b5c16a2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/9d9244ac5bfb4763a3671e3d883e8651] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=35.5 K 2024-12-09T20:44:31,533 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d5fcdda8c04c0ca776c55cd6916884, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733777066930 2024-12-09T20:44:31,533 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54c4dcc8e1394315bea8fd4813260d8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733777066804 2024-12-09T20:44:31,533 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba76249f5684f4ab8c2034c0e77f249, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733777068088 2024-12-09T20:44:31,533 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cd5400576324f11a68af9dc4b5c16a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733777066930 2024-12-09T20:44:31,533 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d9244ac5bfb4763a3671e3d883e8651, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733777068098 2024-12-09T20:44:31,537 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:31,539 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120983e98292de164a8a9f5ad9231a0f4561_b46d5304354f6bfcad03af835fcf16f1 store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:31,540 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#B#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:31,540 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120983e98292de164a8a9f5ad9231a0f4561_b46d5304354f6bfcad03af835fcf16f1, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:31,540 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120983e98292de164a8a9f5ad9231a0f4561_b46d5304354f6bfcad03af835fcf16f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:31,540 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/5e675a09db604d41a5a6a2943d78abbd is 50, key is test_row_0/B:col10/1733777068098/Put/seqid=0 2024-12-09T20:44:31,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742389_1565 (size=12459) 2024-12-09T20:44:31,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742390_1566 (size=4469) 2024-12-09T20:44:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T20:44:31,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-09T20:44:31,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:31,626 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T20:44:31,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:31,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:31,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:31,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:31,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:31,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:31,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209296d46a29d4a43ddb68d7c94a127609b_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777070318/Put/seqid=0 2024-12-09T20:44:31,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742391_1567 (size=12304) 2024-12-09T20:44:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T20:44:31,948 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#A#compaction#480 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:31,948 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/7a90ff3d74dc42478e58dc66f457b72b is 175, key is test_row_0/A:col10/1733777068098/Put/seqid=0 2024-12-09T20:44:31,952 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/5e675a09db604d41a5a6a2943d78abbd as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5e675a09db604d41a5a6a2943d78abbd 2024-12-09T20:44:31,956 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/B of b46d5304354f6bfcad03af835fcf16f1 into 5e675a09db604d41a5a6a2943d78abbd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:31,956 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:31,956 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/B, priority=13, startTime=1733777071531; duration=0sec 2024-12-09T20:44:31,956 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:31,956 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:B 2024-12-09T20:44:31,956 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:31,957 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:31,957 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/C is initiating minor compaction (all files) 2024-12-09T20:44:31,957 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/C in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:31,957 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/0e90a816693c4fba888104e386e460d1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/770d6a6634324abbad18e6d60f622dc1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/310ccc319d5f4311ab832aaee231127e] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=35.5 K 2024-12-09T20:44:31,957 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e90a816693c4fba888104e386e460d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733777066804 2024-12-09T20:44:31,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742392_1568 (size=31413) 2024-12-09T20:44:31,958 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 770d6a6634324abbad18e6d60f622dc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733777066930 2024-12-09T20:44:31,958 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 310ccc319d5f4311ab832aaee231127e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733777068098 2024-12-09T20:44:31,964 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/7a90ff3d74dc42478e58dc66f457b72b as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7a90ff3d74dc42478e58dc66f457b72b 2024-12-09T20:44:31,969 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/A of b46d5304354f6bfcad03af835fcf16f1 into 7a90ff3d74dc42478e58dc66f457b72b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:31,969 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:31,969 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/A, priority=13, startTime=1733777071531; duration=0sec 2024-12-09T20:44:31,969 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:31,969 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:A 2024-12-09T20:44:31,971 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:31,972 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/c92c8608c37a4f7bab00c2b61e774992 is 50, key is test_row_0/C:col10/1733777068098/Put/seqid=0 2024-12-09T20:44:31,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742393_1569 (size=12459) 2024-12-09T20:44:32,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:32,038 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209296d46a29d4a43ddb68d7c94a127609b_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209296d46a29d4a43ddb68d7c94a127609b_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/8b433149c2c648a19ee1ad4488713cab, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:32,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/8b433149c2c648a19ee1ad4488713cab is 175, key is test_row_0/A:col10/1733777070318/Put/seqid=0 2024-12-09T20:44:32,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742394_1570 (size=31105) 2024-12-09T20:44:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T20:44:32,382 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/c92c8608c37a4f7bab00c2b61e774992 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c92c8608c37a4f7bab00c2b61e774992 2024-12-09T20:44:32,388 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/C of b46d5304354f6bfcad03af835fcf16f1 into c92c8608c37a4f7bab00c2b61e774992(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:32,388 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:32,388 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/C, priority=13, startTime=1733777071531; duration=0sec 2024-12-09T20:44:32,389 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:32,389 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:C 2024-12-09T20:44:32,442 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/8b433149c2c648a19ee1ad4488713cab 2024-12-09T20:44:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:32,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:32,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/c8f3d488cfea4e3c979bbfecaef29617 is 50, key is test_row_0/B:col10/1733777070318/Put/seqid=0 2024-12-09T20:44:32,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742395_1571 (size=12151) 2024-12-09T20:44:32,469 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/c8f3d488cfea4e3c979bbfecaef29617 2024-12-09T20:44:32,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/9029608684f046139e781d25bcd007d3 is 50, key is test_row_0/C:col10/1733777070318/Put/seqid=0 2024-12-09T20:44:32,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742396_1572 (size=12151) 2024-12-09T20:44:32,485 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/9029608684f046139e781d25bcd007d3 2024-12-09T20:44:32,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/8b433149c2c648a19ee1ad4488713cab as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/8b433149c2c648a19ee1ad4488713cab 2024-12-09T20:44:32,498 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/8b433149c2c648a19ee1ad4488713cab, entries=150, sequenceid=153, filesize=30.4 K 2024-12-09T20:44:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777132494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/c8f3d488cfea4e3c979bbfecaef29617 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c8f3d488cfea4e3c979bbfecaef29617 2024-12-09T20:44:32,502 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c8f3d488cfea4e3c979bbfecaef29617, entries=150, sequenceid=153, filesize=11.9 K 2024-12-09T20:44:32,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777132495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/9029608684f046139e781d25bcd007d3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/9029608684f046139e781d25bcd007d3 2024-12-09T20:44:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777132499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777132494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,505 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/9029608684f046139e781d25bcd007d3, entries=150, sequenceid=153, filesize=11.9 K 2024-12-09T20:44:32,506 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for b46d5304354f6bfcad03af835fcf16f1 in 880ms, sequenceid=153, compaction requested=false 2024-12-09T20:44:32,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:32,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:32,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-09T20:44:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-09T20:44:32,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-09T20:44:32,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-12-09T20:44:32,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.0390 sec 2024-12-09T20:44:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:32,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:44:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:32,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:32,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094d20382b80af426eba83346db21d2620_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777072476/Put/seqid=0 2024-12-09T20:44:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T20:44:32,611 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:32,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-09T20:44:32,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T20:44:32,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:32,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777132617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777132617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777132618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777132619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742397_1573 (size=14794) 2024-12-09T20:44:32,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:32,629 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094d20382b80af426eba83346db21d2620_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094d20382b80af426eba83346db21d2620_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:32,630 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/f87b4c6ac4564d1eb0cebbaef2416912, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:32,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/f87b4c6ac4564d1eb0cebbaef2416912 is 175, key is test_row_0/A:col10/1733777072476/Put/seqid=0 2024-12-09T20:44:32,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742398_1574 (size=39749) 2024-12-09T20:44:32,635 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/f87b4c6ac4564d1eb0cebbaef2416912 2024-12-09T20:44:32,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/758eb3f6848b46098843457455e634ac is 50, key is test_row_0/B:col10/1733777072476/Put/seqid=0 2024-12-09T20:44:32,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742399_1575 (size=12151) 2024-12-09T20:44:32,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/758eb3f6848b46098843457455e634ac 2024-12-09T20:44:32,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/5ca39a7d4de8425f8e1fb4c32d11e8e6 is 50, key is test_row_0/C:col10/1733777072476/Put/seqid=0 2024-12-09T20:44:32,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742400_1576 (size=12151) 2024-12-09T20:44:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T20:44:32,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777132731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777132731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777132731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777132732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T20:44:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:32,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:32,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T20:44:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:32,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T20:44:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777132941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777132941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777132941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777132941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/5ca39a7d4de8425f8e1fb4c32d11e8e6 2024-12-09T20:44:33,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/f87b4c6ac4564d1eb0cebbaef2416912 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/f87b4c6ac4564d1eb0cebbaef2416912 2024-12-09T20:44:33,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/f87b4c6ac4564d1eb0cebbaef2416912, entries=200, sequenceid=175, filesize=38.8 K 2024-12-09T20:44:33,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/758eb3f6848b46098843457455e634ac as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/758eb3f6848b46098843457455e634ac 2024-12-09T20:44:33,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/758eb3f6848b46098843457455e634ac, entries=150, sequenceid=175, filesize=11.9 K 2024-12-09T20:44:33,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/5ca39a7d4de8425f8e1fb4c32d11e8e6 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/5ca39a7d4de8425f8e1fb4c32d11e8e6 2024-12-09T20:44:33,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T20:44:33,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/5ca39a7d4de8425f8e1fb4c32d11e8e6, entries=150, sequenceid=175, filesize=11.9 K 2024-12-09T20:44:33,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:33,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:33,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:33,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:33,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:33,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b46d5304354f6bfcad03af835fcf16f1 in 468ms, sequenceid=175, compaction requested=true 2024-12-09T20:44:33,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:33,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:44:33,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:33,073 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:33,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:44:33,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:33,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:44:33,073 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:33,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:33,073 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:33,073 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:33,073 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/B is initiating minor compaction (all files) 2024-12-09T20:44:33,073 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/A is initiating minor compaction (all files) 2024-12-09T20:44:33,073 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/B in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:33,073 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/A in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:33,074 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7a90ff3d74dc42478e58dc66f457b72b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/8b433149c2c648a19ee1ad4488713cab, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/f87b4c6ac4564d1eb0cebbaef2416912] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=99.9 K 2024-12-09T20:44:33,074 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5e675a09db604d41a5a6a2943d78abbd, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c8f3d488cfea4e3c979bbfecaef29617, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/758eb3f6848b46098843457455e634ac] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=35.9 K 2024-12-09T20:44:33,074 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:33,074 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7a90ff3d74dc42478e58dc66f457b72b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/8b433149c2c648a19ee1ad4488713cab, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/f87b4c6ac4564d1eb0cebbaef2416912] 2024-12-09T20:44:33,074 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a90ff3d74dc42478e58dc66f457b72b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733777068098 2024-12-09T20:44:33,074 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e675a09db604d41a5a6a2943d78abbd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733777068098 2024-12-09T20:44:33,074 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f3d488cfea4e3c979bbfecaef29617, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733777070272 2024-12-09T20:44:33,074 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b433149c2c648a19ee1ad4488713cab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733777070272 2024-12-09T20:44:33,075 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting f87b4c6ac4564d1eb0cebbaef2416912, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733777072476 2024-12-09T20:44:33,075 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 758eb3f6848b46098843457455e634ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733777072476 2024-12-09T20:44:33,079 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:33,081 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209febc63c217194ebaa54dedc1dcf3dd77_b46d5304354f6bfcad03af835fcf16f1 store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:33,081 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#B#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:33,081 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/63ba642a7edd42efb32569ca37651648 is 50, key is test_row_0/B:col10/1733777072476/Put/seqid=0 2024-12-09T20:44:33,082 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209febc63c217194ebaa54dedc1dcf3dd77_b46d5304354f6bfcad03af835fcf16f1, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:33,082 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209febc63c217194ebaa54dedc1dcf3dd77_b46d5304354f6bfcad03af835fcf16f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:33,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742401_1577 (size=12561) 2024-12-09T20:44:33,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742402_1578 (size=4469) 2024-12-09T20:44:33,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T20:44:33,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:33,224 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:44:33,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:33,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:33,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:33,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:33,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:33,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:33,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e11b0270c47246c9aefd17dd70f35d1e_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777072618/Put/seqid=0 2024-12-09T20:44:33,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742403_1579 (size=12304) 2024-12-09T20:44:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T20:44:33,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:33,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:33,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777133269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777133270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777133272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777133273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777133381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777133381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777133381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777133381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,501 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/63ba642a7edd42efb32569ca37651648 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/63ba642a7edd42efb32569ca37651648 2024-12-09T20:44:33,502 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#A#compaction#490 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:33,503 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/75b0d2f1303e433c8ca891b05d3bf08d is 175, key is test_row_0/A:col10/1733777072476/Put/seqid=0 2024-12-09T20:44:33,504 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/B of b46d5304354f6bfcad03af835fcf16f1 into 63ba642a7edd42efb32569ca37651648(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:33,504 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:33,504 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/B, priority=13, startTime=1733777073073; duration=0sec 2024-12-09T20:44:33,504 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:33,504 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:B 2024-12-09T20:44:33,504 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:33,505 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:33,505 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/C is initiating minor compaction (all files) 2024-12-09T20:44:33,505 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/C in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:33,505 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c92c8608c37a4f7bab00c2b61e774992, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/9029608684f046139e781d25bcd007d3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/5ca39a7d4de8425f8e1fb4c32d11e8e6] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=35.9 K 2024-12-09T20:44:33,506 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c92c8608c37a4f7bab00c2b61e774992, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733777068098 2024-12-09T20:44:33,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742404_1580 (size=31515) 2024-12-09T20:44:33,506 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 9029608684f046139e781d25bcd007d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733777070272 2024-12-09T20:44:33,506 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca39a7d4de8425f8e1fb4c32d11e8e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733777072476 2024-12-09T20:44:33,511 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:33,512 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/7fb46c04f40b4b70b25d7f4751a68316 is 50, key is test_row_0/C:col10/1733777072476/Put/seqid=0 2024-12-09T20:44:33,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742405_1581 (size=12561) 2024-12-09T20:44:33,517 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/7fb46c04f40b4b70b25d7f4751a68316 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7fb46c04f40b4b70b25d7f4751a68316 2024-12-09T20:44:33,521 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/C of b46d5304354f6bfcad03af835fcf16f1 into 7fb46c04f40b4b70b25d7f4751a68316(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:33,521 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:33,521 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/C, priority=13, startTime=1733777073073; duration=0sec 2024-12-09T20:44:33,521 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:33,521 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:C 2024-12-09T20:44:33,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777133591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777133591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777133591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777133591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:33,635 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e11b0270c47246c9aefd17dd70f35d1e_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e11b0270c47246c9aefd17dd70f35d1e_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:33,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/02eaf79767c142fd8f4a0bd98de55f6d, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:33,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/02eaf79767c142fd8f4a0bd98de55f6d is 175, key is test_row_0/A:col10/1733777072618/Put/seqid=0 2024-12-09T20:44:33,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742406_1582 (size=31105) 2024-12-09T20:44:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T20:44:33,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777133901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777133901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777133901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777133901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:33,909 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/75b0d2f1303e433c8ca891b05d3bf08d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/75b0d2f1303e433c8ca891b05d3bf08d 2024-12-09T20:44:33,913 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/A of b46d5304354f6bfcad03af835fcf16f1 into 75b0d2f1303e433c8ca891b05d3bf08d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:33,913 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:33,913 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/A, priority=13, startTime=1733777073072; duration=0sec 2024-12-09T20:44:33,913 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:33,913 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:A 2024-12-09T20:44:34,039 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/02eaf79767c142fd8f4a0bd98de55f6d 2024-12-09T20:44:34,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/70f40bb3d72b472e81a16dc56d90f5ec is 50, key is test_row_0/B:col10/1733777072618/Put/seqid=0 2024-12-09T20:44:34,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742407_1583 (size=12151) 2024-12-09T20:44:34,050 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/70f40bb3d72b472e81a16dc56d90f5ec 2024-12-09T20:44:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/e156cb5395ed400f94f78111a3c85060 is 50, key is test_row_0/C:col10/1733777072618/Put/seqid=0 2024-12-09T20:44:34,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742408_1584 (size=12151) 2024-12-09T20:44:34,114 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T20:44:34,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777134411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:34,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777134412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:34,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777134412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:34,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777134412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:34,462 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/e156cb5395ed400f94f78111a3c85060 2024-12-09T20:44:34,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/02eaf79767c142fd8f4a0bd98de55f6d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/02eaf79767c142fd8f4a0bd98de55f6d 2024-12-09T20:44:34,468 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/02eaf79767c142fd8f4a0bd98de55f6d, entries=150, sequenceid=188, filesize=30.4 K 2024-12-09T20:44:34,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/70f40bb3d72b472e81a16dc56d90f5ec as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/70f40bb3d72b472e81a16dc56d90f5ec 2024-12-09T20:44:34,471 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/70f40bb3d72b472e81a16dc56d90f5ec, entries=150, sequenceid=188, filesize=11.9 K 2024-12-09T20:44:34,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/e156cb5395ed400f94f78111a3c85060 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/e156cb5395ed400f94f78111a3c85060 2024-12-09T20:44:34,474 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/e156cb5395ed400f94f78111a3c85060, entries=150, sequenceid=188, filesize=11.9 K 2024-12-09T20:44:34,475 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b46d5304354f6bfcad03af835fcf16f1 in 1250ms, sequenceid=188, compaction requested=false 2024-12-09T20:44:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-09T20:44:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-09T20:44:34,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-09T20:44:34,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-12-09T20:44:34,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8670 sec 2024-12-09T20:44:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T20:44:34,761 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:34,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-09T20:44:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T20:44:34,763 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:34,764 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:34,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T20:44:34,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T20:44:34,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:34,916 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T20:44:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:34,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098cfbde579ab84c57a2fe5de7d73487ba_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777073254/Put/seqid=0 2024-12-09T20:44:34,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742409_1585 (size=12304) 2024-12-09T20:44:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T20:44:35,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:35,330 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098cfbde579ab84c57a2fe5de7d73487ba_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098cfbde579ab84c57a2fe5de7d73487ba_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/18a206f1c66148b0bb036ec3195237bc, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/18a206f1c66148b0bb036ec3195237bc is 175, key is test_row_0/A:col10/1733777073254/Put/seqid=0 2024-12-09T20:44:35,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742410_1586 (size=31105) 2024-12-09T20:44:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T20:44:35,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:35,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777135452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777135453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777135456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777135458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777135561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777135561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777135561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777135572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,744 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/18a206f1c66148b0bb036ec3195237bc 2024-12-09T20:44:35,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/5cf4a088215149f9b67b3f189bf03375 is 50, key is test_row_0/B:col10/1733777073254/Put/seqid=0 2024-12-09T20:44:35,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777135771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777135772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777135772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777135792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:35,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742411_1587 (size=12151) 2024-12-09T20:44:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T20:44:36,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777136081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777136081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777136081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:36,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777136101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:36,221 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/5cf4a088215149f9b67b3f189bf03375 2024-12-09T20:44:36,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/ea5670bedf60422ca4237aa6a7ee2746 is 50, key is test_row_0/C:col10/1733777073254/Put/seqid=0 2024-12-09T20:44:36,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742412_1588 (size=12151) 2024-12-09T20:44:36,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777136591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:36,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777136591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:36,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777136591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:36,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777136611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:36,631 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/ea5670bedf60422ca4237aa6a7ee2746 2024-12-09T20:44:36,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/18a206f1c66148b0bb036ec3195237bc as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/18a206f1c66148b0bb036ec3195237bc 2024-12-09T20:44:36,639 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/18a206f1c66148b0bb036ec3195237bc, entries=150, sequenceid=210, filesize=30.4 K 2024-12-09T20:44:36,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/5cf4a088215149f9b67b3f189bf03375 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5cf4a088215149f9b67b3f189bf03375 2024-12-09T20:44:36,642 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5cf4a088215149f9b67b3f189bf03375, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T20:44:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/ea5670bedf60422ca4237aa6a7ee2746 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/ea5670bedf60422ca4237aa6a7ee2746 2024-12-09T20:44:36,645 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/ea5670bedf60422ca4237aa6a7ee2746, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T20:44:36,646 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b46d5304354f6bfcad03af835fcf16f1 in 1731ms, sequenceid=210, compaction requested=true 2024-12-09T20:44:36,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:36,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:36,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-09T20:44:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-09T20:44:36,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-09T20:44:36,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8830 sec 2024-12-09T20:44:36,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8880 sec 2024-12-09T20:44:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T20:44:36,911 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:36,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:36,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-09T20:44:36,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T20:44:36,913 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:36,914 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:36,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T20:44:37,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T20:44:37,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:37,066 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T20:44:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:37,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120960a7e792fcfc46589b2badd95e296fe6_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777075438/Put/seqid=0 2024-12-09T20:44:37,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742413_1589 (size=12304) 2024-12-09T20:44:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T20:44:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:37,477 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120960a7e792fcfc46589b2badd95e296fe6_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120960a7e792fcfc46589b2badd95e296fe6_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/407aa1a62648485ea5249a5512b1e376, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:37,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/407aa1a62648485ea5249a5512b1e376 is 175, key is test_row_0/A:col10/1733777075438/Put/seqid=0 2024-12-09T20:44:37,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742414_1590 (size=31105) 2024-12-09T20:44:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T20:44:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:37,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777137626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777137627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777137627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777137628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777137741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777137741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777137741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777137741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,881 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/407aa1a62648485ea5249a5512b1e376 2024-12-09T20:44:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/4dd07499f9ab4d72a7ce54b7e2786c11 is 50, key is test_row_0/B:col10/1733777075438/Put/seqid=0 2024-12-09T20:44:37,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742415_1591 (size=12151) 2024-12-09T20:44:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777137951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777137951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777137951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777137951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T20:44:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777138261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777138261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777138262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777138262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,288 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/4dd07499f9ab4d72a7ce54b7e2786c11 2024-12-09T20:44:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/b3077a6e549546eeba4379acf92bce4c is 50, key is test_row_0/C:col10/1733777075438/Put/seqid=0 2024-12-09T20:44:38,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742416_1592 (size=12151) 2024-12-09T20:44:38,696 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/b3077a6e549546eeba4379acf92bce4c 2024-12-09T20:44:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/407aa1a62648485ea5249a5512b1e376 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/407aa1a62648485ea5249a5512b1e376 2024-12-09T20:44:38,704 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/407aa1a62648485ea5249a5512b1e376, entries=150, sequenceid=223, filesize=30.4 K 2024-12-09T20:44:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/4dd07499f9ab4d72a7ce54b7e2786c11 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4dd07499f9ab4d72a7ce54b7e2786c11 2024-12-09T20:44:38,708 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4dd07499f9ab4d72a7ce54b7e2786c11, entries=150, sequenceid=223, filesize=11.9 K 2024-12-09T20:44:38,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/b3077a6e549546eeba4379acf92bce4c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b3077a6e549546eeba4379acf92bce4c 2024-12-09T20:44:38,712 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b3077a6e549546eeba4379acf92bce4c, entries=150, sequenceid=223, filesize=11.9 K 2024-12-09T20:44:38,713 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b46d5304354f6bfcad03af835fcf16f1 in 1647ms, sequenceid=223, compaction requested=true 2024-12-09T20:44:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-09T20:44:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-09T20:44:38,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-09T20:44:38,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-12-09T20:44:38,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.8060 sec 2024-12-09T20:44:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:38,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T20:44:38,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:38,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:38,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:38,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:38,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:38,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T20:44:38,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120987406c7d9cb642269acc69732083d563_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777077625/Put/seqid=0 2024-12-09T20:44:38,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777138786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777138787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777138787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777138788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742417_1593 (size=12304) 2024-12-09T20:44:38,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777138901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777138901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777138901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777138901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T20:44:39,062 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:39,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T20:44:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-09T20:44:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T20:44:39,064 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T20:44:39,064 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T20:44:39,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T20:44:39,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777139111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777139111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777139111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777139111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T20:44:39,195 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:39,198 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120987406c7d9cb642269acc69732083d563_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120987406c7d9cb642269acc69732083d563_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:39,198 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/7cc710e3df7a4e6a8e3fb1a56aa835b7, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:39,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/7cc710e3df7a4e6a8e3fb1a56aa835b7 is 175, key is test_row_0/A:col10/1733777077625/Put/seqid=0 2024-12-09T20:44:39,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742418_1594 (size=31105) 2024-12-09T20:44:39,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T20:44:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T20:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T20:44:39,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777139421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777139421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777139421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777139421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T20:44:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48658 deadline: 1733777139571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,574 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 00c23a7251f8,36519,1733776895755 for 'test_row_2' in b46d5304354f6bfcad03af835fcf16f1 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18378 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T20:44:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,602 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/7cc710e3df7a4e6a8e3fb1a56aa835b7 2024-12-09T20:44:39,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/c17a49db74344110aae5f53299a95619 is 50, key is test_row_0/B:col10/1733777077625/Put/seqid=0 2024-12-09T20:44:39,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742419_1595 (size=12151) 2024-12-09T20:44:39,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T20:44:39,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:39,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T20:44:39,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T20:44:39,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:39,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777139931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777139931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777139931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777139931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:39,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T20:44:39,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:39,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:39,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:40,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/c17a49db74344110aae5f53299a95619 2024-12-09T20:44:40,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/b16c9d45f0fa4d06aaf646a367bdf153 is 50, key is test_row_0/C:col10/1733777077625/Put/seqid=0 2024-12-09T20:44:40,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742420_1596 (size=12151) 2024-12-09T20:44:40,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T20:44:40,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:40,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:40,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:40,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:40,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T20:44:40,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T20:44:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:40,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:44:40,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/b16c9d45f0fa4d06aaf646a367bdf153 2024-12-09T20:44:40,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/7cc710e3df7a4e6a8e3fb1a56aa835b7 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7cc710e3df7a4e6a8e3fb1a56aa835b7 2024-12-09T20:44:40,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7cc710e3df7a4e6a8e3fb1a56aa835b7, entries=150, sequenceid=244, filesize=30.4 K 2024-12-09T20:44:40,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/c17a49db74344110aae5f53299a95619 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c17a49db74344110aae5f53299a95619 2024-12-09T20:44:40,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c17a49db74344110aae5f53299a95619, entries=150, sequenceid=244, filesize=11.9 K 2024-12-09T20:44:40,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/b16c9d45f0fa4d06aaf646a367bdf153 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b16c9d45f0fa4d06aaf646a367bdf153 2024-12-09T20:44:40,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b16c9d45f0fa4d06aaf646a367bdf153, entries=150, sequenceid=244, filesize=11.9 K 2024-12-09T20:44:40,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b46d5304354f6bfcad03af835fcf16f1 in 1652ms, sequenceid=244, compaction requested=true 2024-12-09T20:44:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:44:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:40,428 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T20:44:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:44:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:40,428 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T20:44:40,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:44:40,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:40,432 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 155935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T20:44:40,432 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T20:44:40,432 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/A is initiating minor compaction (all files) 2024-12-09T20:44:40,432 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/B is initiating minor compaction (all files) 2024-12-09T20:44:40,432 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/B in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:40,432 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/A in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:40,432 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/75b0d2f1303e433c8ca891b05d3bf08d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/02eaf79767c142fd8f4a0bd98de55f6d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/18a206f1c66148b0bb036ec3195237bc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/407aa1a62648485ea5249a5512b1e376, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7cc710e3df7a4e6a8e3fb1a56aa835b7] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=152.3 K 2024-12-09T20:44:40,432 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/63ba642a7edd42efb32569ca37651648, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/70f40bb3d72b472e81a16dc56d90f5ec, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5cf4a088215149f9b67b3f189bf03375, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4dd07499f9ab4d72a7ce54b7e2786c11, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c17a49db74344110aae5f53299a95619] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=59.7 K 2024-12-09T20:44:40,432 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:40,432 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/75b0d2f1303e433c8ca891b05d3bf08d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/02eaf79767c142fd8f4a0bd98de55f6d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/18a206f1c66148b0bb036ec3195237bc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/407aa1a62648485ea5249a5512b1e376, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7cc710e3df7a4e6a8e3fb1a56aa835b7] 2024-12-09T20:44:40,432 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 63ba642a7edd42efb32569ca37651648, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733777072476 2024-12-09T20:44:40,432 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b0d2f1303e433c8ca891b05d3bf08d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733777072476 2024-12-09T20:44:40,433 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 70f40bb3d72b472e81a16dc56d90f5ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733777072615 2024-12-09T20:44:40,433 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02eaf79767c142fd8f4a0bd98de55f6d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733777072615 2024-12-09T20:44:40,433 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf4a088215149f9b67b3f189bf03375, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733777073254 2024-12-09T20:44:40,433 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18a206f1c66148b0bb036ec3195237bc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733777073254 2024-12-09T20:44:40,433 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd07499f9ab4d72a7ce54b7e2786c11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733777075438 2024-12-09T20:44:40,433 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 407aa1a62648485ea5249a5512b1e376, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733777075438 2024-12-09T20:44:40,433 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c17a49db74344110aae5f53299a95619, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777077625 2024-12-09T20:44:40,433 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc710e3df7a4e6a8e3fb1a56aa835b7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777077625 2024-12-09T20:44:40,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36519 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T20:44:40,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:40,437 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T20:44:40,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:40,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:40,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:40,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:40,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:40,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:40,443 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#B#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:40,443 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/95c09d7d59f945c089938223988259ea is 50, key is test_row_0/B:col10/1733777077625/Put/seqid=0 2024-12-09T20:44:40,444 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:40,448 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209c17db9d9087745cda36689a75693d518_b46d5304354f6bfcad03af835fcf16f1 store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:40,449 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209c17db9d9087745cda36689a75693d518_b46d5304354f6bfcad03af835fcf16f1, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:40,449 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c17db9d9087745cda36689a75693d518_b46d5304354f6bfcad03af835fcf16f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209870b974c0c2d4b0caa16cfd8bdf567d8_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777078786/Put/seqid=0 2024-12-09T20:44:40,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742421_1597 (size=12731) 2024-12-09T20:44:40,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742422_1598 (size=4469) 2024-12-09T20:44:40,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742423_1599 (size=12304) 2024-12-09T20:44:40,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:40,480 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209870b974c0c2d4b0caa16cfd8bdf567d8_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209870b974c0c2d4b0caa16cfd8bdf567d8_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/01a2f47eb18a4ad3a585ade2f9747d6d, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:40,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/01a2f47eb18a4ad3a585ade2f9747d6d is 175, key is test_row_0/A:col10/1733777078786/Put/seqid=0 2024-12-09T20:44:40,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742424_1600 (size=31105) 2024-12-09T20:44:40,485 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/01a2f47eb18a4ad3a585ade2f9747d6d 2024-12-09T20:44:40,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/e1cbd8525b614e7baf835a935809cde2 is 50, key is test_row_0/B:col10/1733777078786/Put/seqid=0 2024-12-09T20:44:40,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742425_1601 (size=12151) 2024-12-09T20:44:40,861 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/95c09d7d59f945c089938223988259ea as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/95c09d7d59f945c089938223988259ea 2024-12-09T20:44:40,864 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/B of b46d5304354f6bfcad03af835fcf16f1 into 95c09d7d59f945c089938223988259ea(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:40,864 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:40,864 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/B, priority=11, startTime=1733777080428; duration=0sec 2024-12-09T20:44:40,864 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:40,864 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:B 2024-12-09T20:44:40,864 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T20:44:40,865 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T20:44:40,865 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/C is initiating minor compaction (all files) 2024-12-09T20:44:40,865 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/C in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:40,866 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7fb46c04f40b4b70b25d7f4751a68316, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/e156cb5395ed400f94f78111a3c85060, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/ea5670bedf60422ca4237aa6a7ee2746, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b3077a6e549546eeba4379acf92bce4c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b16c9d45f0fa4d06aaf646a367bdf153] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=59.7 K 2024-12-09T20:44:40,866 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fb46c04f40b4b70b25d7f4751a68316, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733777072476 2024-12-09T20:44:40,866 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting e156cb5395ed400f94f78111a3c85060, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733777072615 2024-12-09T20:44:40,866 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting ea5670bedf60422ca4237aa6a7ee2746, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733777073254 2024-12-09T20:44:40,867 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b3077a6e549546eeba4379acf92bce4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733777075438 2024-12-09T20:44:40,867 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting b16c9d45f0fa4d06aaf646a367bdf153, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777077625 2024-12-09T20:44:40,869 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#A#compaction#505 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:40,870 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/fa623ae2cca144f5b52cd40e33fa0f47 is 175, key is test_row_0/A:col10/1733777077625/Put/seqid=0 2024-12-09T20:44:40,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742426_1602 (size=31685) 2024-12-09T20:44:40,877 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#C#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:40,877 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/7330440b8dee41599eac0479dce6de7e is 50, key is test_row_0/C:col10/1733777077625/Put/seqid=0 2024-12-09T20:44:40,880 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/fa623ae2cca144f5b52cd40e33fa0f47 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/fa623ae2cca144f5b52cd40e33fa0f47 2024-12-09T20:44:40,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742427_1603 (size=12731) 2024-12-09T20:44:40,884 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/7330440b8dee41599eac0479dce6de7e as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7330440b8dee41599eac0479dce6de7e 2024-12-09T20:44:40,885 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/A of b46d5304354f6bfcad03af835fcf16f1 into fa623ae2cca144f5b52cd40e33fa0f47(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:40,885 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:40,885 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/A, priority=11, startTime=1733777080428; duration=0sec 2024-12-09T20:44:40,885 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:40,885 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:A 2024-12-09T20:44:40,887 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/C of b46d5304354f6bfcad03af835fcf16f1 into 7330440b8dee41599eac0479dce6de7e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:40,887 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:40,887 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/C, priority=11, startTime=1733777080428; duration=0sec 2024-12-09T20:44:40,888 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:40,888 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:C 2024-12-09T20:44:40,897 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/e1cbd8525b614e7baf835a935809cde2 2024-12-09T20:44:40,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/012a179bf7cd4da286aeb152fcc03b3f is 50, key is test_row_0/C:col10/1733777078786/Put/seqid=0 2024-12-09T20:44:40,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742428_1604 (size=12151) 2024-12-09T20:44:40,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. as already flushing 2024-12-09T20:44:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:40,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777080959, max timestamp=1733777080964Num uniques 859; 2024-12-09T20:44:40,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:44:40,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777080959, max timestamp=1733777080964Num uniques 859; 2024-12-09T20:44:40,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:44:40,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733777080959, max timestamp=1733777080964Num uniques 859; 2024-12-09T20:44:40,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/00c23a7251f8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T20:44:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777140981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:40,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777140982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:40,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777140985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:40,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777140986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:41,041 INFO [Thread-2377 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2377. 2024-12-09T20:44:41,041 DEBUG [Thread-2377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:41,041 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:41,042 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,042 INFO [Thread-2381 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2381. 2024-12-09T20:44:41,042 DEBUG [Thread-2381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:41,042 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,042 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:41,043 INFO [Thread-2379 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2379. 2024-12-09T20:44:41,043 DEBUG [Thread-2379 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:41,043 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,043 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:41,046 INFO [Thread-2375 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2375. 2024-12-09T20:44:41,046 INFO [Thread-2383 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2383. 2024-12-09T20:44:41,046 DEBUG [Thread-2383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:41,046 DEBUG [Thread-2375 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:41,047 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,047 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,047 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,047 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:41,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:41,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:41,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777141091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777141091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:41,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:41,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777141101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777141101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T20:44:41,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:41,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48738 deadline: 1733777141302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48700 deadline: 1733777141302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:41,307 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/012a179bf7cd4da286aeb152fcc03b3f 2024-12-09T20:44:41,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:41,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T20:44:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48754 deadline: 1733777141311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36519 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48682 deadline: 1733777141312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 , the old value is region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T20:44:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., hostname=00c23a7251f8,36519,1733776895755, seqNum=5 because the exception is null or not the one we care about 2024-12-09T20:44:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/01a2f47eb18a4ad3a585ade2f9747d6d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/01a2f47eb18a4ad3a585ade2f9747d6d 2024-12-09T20:44:41,321 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/01a2f47eb18a4ad3a585ade2f9747d6d, entries=150, sequenceid=255, filesize=30.4 K 2024-12-09T20:44:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/e1cbd8525b614e7baf835a935809cde2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/e1cbd8525b614e7baf835a935809cde2 2024-12-09T20:44:41,326 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/e1cbd8525b614e7baf835a935809cde2, entries=150, sequenceid=255, filesize=11.9 K 2024-12-09T20:44:41,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/012a179bf7cd4da286aeb152fcc03b3f as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/012a179bf7cd4da286aeb152fcc03b3f 2024-12-09T20:44:41,331 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/012a179bf7cd4da286aeb152fcc03b3f, entries=150, sequenceid=255, filesize=11.9 K 2024-12-09T20:44:41,332 INFO [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b46d5304354f6bfcad03af835fcf16f1 in 895ms, sequenceid=255, compaction requested=false 2024-12-09T20:44:41,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c23a7251f8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-09T20:44:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-09T20:44:41,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-09T20:44:41,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2700 sec 2024-12-09T20:44:41,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.2740 sec 2024-12-09T20:44:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36519 {}] regionserver.HRegion(8855): Flush requested on b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:41,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-09T20:44:41,612 INFO [Thread-2372 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2372. 2024-12-09T20:44:41,612 INFO [Thread-2370 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2370. 2024-12-09T20:44:41,612 DEBUG [Thread-2370 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:41,612 DEBUG [Thread-2372 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:41,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:41,612 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,612 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:44:41,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:41,612 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,612 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:44:41,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:41,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T20:44:41,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:41,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:41,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093dba33f097044bd49ac74e042bb76f1f_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777081611/Put/seqid=0 2024-12-09T20:44:41,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742429_1605 (size=12454) 2024-12-09T20:44:41,622 INFO [Thread-2368 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2368. 2024-12-09T20:44:41,622 DEBUG [Thread-2368 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:41,622 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,622 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:41,622 INFO [Thread-2366 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2366. 2024-12-09T20:44:41,622 DEBUG [Thread-2366 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:41,623 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,623 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:41,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:42,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:42,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093dba33f097044bd49ac74e042bb76f1f_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093dba33f097044bd49ac74e042bb76f1f_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:42,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/33910e8357984118bcc6864f6b5a65f3, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:42,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/33910e8357984118bcc6864f6b5a65f3 is 175, key is test_row_0/A:col10/1733777081611/Put/seqid=0 2024-12-09T20:44:42,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742430_1606 (size=31255) 2024-12-09T20:44:42,432 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/33910e8357984118bcc6864f6b5a65f3 2024-12-09T20:44:42,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/6f8fccc5d02743faa66109c04385e8a5 is 50, key is test_row_0/B:col10/1733777081611/Put/seqid=0 2024-12-09T20:44:42,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742431_1607 (size=12301) 2024-12-09T20:44:42,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/6f8fccc5d02743faa66109c04385e8a5 2024-12-09T20:44:42,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/c313098b01e042e5bbcbe4126434e7e5 is 50, key is test_row_0/C:col10/1733777081611/Put/seqid=0 2024-12-09T20:44:42,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742432_1608 (size=12301) 2024-12-09T20:44:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T20:44:43,253 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:43,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/c313098b01e042e5bbcbe4126434e7e5 2024-12-09T20:44:43,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/33910e8357984118bcc6864f6b5a65f3 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/33910e8357984118bcc6864f6b5a65f3 2024-12-09T20:44:43,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/33910e8357984118bcc6864f6b5a65f3, entries=150, sequenceid=296, filesize=30.5 K 2024-12-09T20:44:43,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/6f8fccc5d02743faa66109c04385e8a5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6f8fccc5d02743faa66109c04385e8a5 2024-12-09T20:44:43,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6f8fccc5d02743faa66109c04385e8a5, entries=150, sequenceid=296, filesize=12.0 K 2024-12-09T20:44:43,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/c313098b01e042e5bbcbe4126434e7e5 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c313098b01e042e5bbcbe4126434e7e5 2024-12-09T20:44:43,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c313098b01e042e5bbcbe4126434e7e5, entries=150, sequenceid=296, filesize=12.0 K 2024-12-09T20:44:43,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=13.42 KB/13740 for b46d5304354f6bfcad03af835fcf16f1 in 1665ms, sequenceid=296, compaction requested=true 2024-12-09T20:44:43,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:43,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T20:44:43,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:43,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T20:44:43,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:43,277 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:43,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b46d5304354f6bfcad03af835fcf16f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T20:44:43,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:43,277 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:43,278 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:43,278 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:43,278 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/A is initiating minor compaction (all files) 2024-12-09T20:44:43,278 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/B is initiating minor compaction (all files) 2024-12-09T20:44:43,278 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/A in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:43,278 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/B in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:43,278 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/fa623ae2cca144f5b52cd40e33fa0f47, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/01a2f47eb18a4ad3a585ade2f9747d6d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/33910e8357984118bcc6864f6b5a65f3] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=91.8 K 2024-12-09T20:44:43,278 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/95c09d7d59f945c089938223988259ea, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/e1cbd8525b614e7baf835a935809cde2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6f8fccc5d02743faa66109c04385e8a5] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=36.3 K 2024-12-09T20:44:43,278 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:43,278 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. files: [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/fa623ae2cca144f5b52cd40e33fa0f47, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/01a2f47eb18a4ad3a585ade2f9747d6d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/33910e8357984118bcc6864f6b5a65f3] 2024-12-09T20:44:43,278 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 95c09d7d59f945c089938223988259ea, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777077625 2024-12-09T20:44:43,278 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa623ae2cca144f5b52cd40e33fa0f47, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777077625 2024-12-09T20:44:43,279 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting e1cbd8525b614e7baf835a935809cde2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733777078781 2024-12-09T20:44:43,279 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01a2f47eb18a4ad3a585ade2f9747d6d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733777078781 2024-12-09T20:44:43,279 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8fccc5d02743faa66109c04385e8a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733777080985 2024-12-09T20:44:43,279 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33910e8357984118bcc6864f6b5a65f3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733777080985 2024-12-09T20:44:43,283 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:43,285 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:43,285 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209c629c4d4a9774b14927e73905141834c_b46d5304354f6bfcad03af835fcf16f1 store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:43,285 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/47620506bc52414690a198799efdcba0 is 50, key is test_row_0/B:col10/1733777081611/Put/seqid=0 2024-12-09T20:44:43,287 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209c629c4d4a9774b14927e73905141834c_b46d5304354f6bfcad03af835fcf16f1, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:43,287 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c629c4d4a9774b14927e73905141834c_b46d5304354f6bfcad03af835fcf16f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:43,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742433_1609 (size=12983) 2024-12-09T20:44:43,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742434_1610 (size=4469) 2024-12-09T20:44:43,696 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:43,697 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/dde494b7708e4b47ae16eaea90c5a64d is 175, key is test_row_0/A:col10/1733777081611/Put/seqid=0 2024-12-09T20:44:43,698 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/47620506bc52414690a198799efdcba0 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/47620506bc52414690a198799efdcba0 2024-12-09T20:44:43,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742435_1611 (size=31937) 2024-12-09T20:44:43,701 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/B of b46d5304354f6bfcad03af835fcf16f1 into 47620506bc52414690a198799efdcba0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:43,701 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:43,701 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/B, priority=13, startTime=1733777083277; duration=0sec 2024-12-09T20:44:43,701 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T20:44:43,701 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:B 2024-12-09T20:44:43,702 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T20:44:43,702 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T20:44:43,702 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1541): b46d5304354f6bfcad03af835fcf16f1/C is initiating minor compaction (all files) 2024-12-09T20:44:43,702 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b46d5304354f6bfcad03af835fcf16f1/C in TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:43,702 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7330440b8dee41599eac0479dce6de7e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/012a179bf7cd4da286aeb152fcc03b3f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c313098b01e042e5bbcbe4126434e7e5] into tmpdir=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp, totalSize=36.3 K 2024-12-09T20:44:43,702 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/dde494b7708e4b47ae16eaea90c5a64d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dde494b7708e4b47ae16eaea90c5a64d 2024-12-09T20:44:43,703 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 7330440b8dee41599eac0479dce6de7e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733777077625 2024-12-09T20:44:43,703 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting 012a179bf7cd4da286aeb152fcc03b3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733777078781 2024-12-09T20:44:43,703 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] compactions.Compactor(225): Compacting c313098b01e042e5bbcbe4126434e7e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733777080985 2024-12-09T20:44:43,706 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/A of b46d5304354f6bfcad03af835fcf16f1 into dde494b7708e4b47ae16eaea90c5a64d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:43,706 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:43,706 INFO [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/A, priority=13, startTime=1733777083277; duration=0sec 2024-12-09T20:44:43,706 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:43,706 DEBUG [RS:0;00c23a7251f8:36519-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:A 2024-12-09T20:44:43,709 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b46d5304354f6bfcad03af835fcf16f1#C#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T20:44:43,709 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/0a9f7b995b264091ad1cd14e1571cb42 is 50, key is test_row_0/C:col10/1733777081611/Put/seqid=0 2024-12-09T20:44:43,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742436_1612 (size=12983) 2024-12-09T20:44:44,122 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/0a9f7b995b264091ad1cd14e1571cb42 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/0a9f7b995b264091ad1cd14e1571cb42 2024-12-09T20:44:44,128 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b46d5304354f6bfcad03af835fcf16f1/C of b46d5304354f6bfcad03af835fcf16f1 into 0a9f7b995b264091ad1cd14e1571cb42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T20:44:44,128 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b46d5304354f6bfcad03af835fcf16f1: 2024-12-09T20:44:44,128 INFO [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., storeName=b46d5304354f6bfcad03af835fcf16f1/C, priority=13, startTime=1733777083277; duration=0sec 2024-12-09T20:44:44,128 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T20:44:44,128 DEBUG [RS:0;00c23a7251f8:36519-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b46d5304354f6bfcad03af835fcf16f1:C 2024-12-09T20:44:49,593 INFO [Thread-2364 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2364. 2024-12-09T20:44:49,593 DEBUG [Thread-2364 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T20:44:49,593 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:49,593 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:49,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4199 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4067 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4023 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4164 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4072 2024-12-09T20:44:49,594 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T20:44:49,594 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:44:49,594 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:44:49,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:49,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:49,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:49,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T20:44:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T20:44:49,599 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777089599"}]},"ts":"1733777089599"} 2024-12-09T20:44:49,601 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T20:44:49,601 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T20:44:49,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T20:44:49,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, UNASSIGN}] 2024-12-09T20:44:49,605 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, UNASSIGN 2024-12-09T20:44:49,606 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=b46d5304354f6bfcad03af835fcf16f1, regionState=CLOSING, regionLocation=00c23a7251f8,36519,1733776895755 2024-12-09T20:44:49,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, UNASSIGN because future has completed 2024-12-09T20:44:49,607 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T20:44:49,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755}] 2024-12-09T20:44:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T20:44:49,761 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:49,761 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T20:44:49,761 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing b46d5304354f6bfcad03af835fcf16f1, disabling compactions & flushes 2024-12-09T20:44:49,761 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:49,761 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:49,761 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. after waiting 0 ms 2024-12-09T20:44:49,761 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:49,761 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing b46d5304354f6bfcad03af835fcf16f1 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-09T20:44:49,762 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=A 2024-12-09T20:44:49,762 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:49,762 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=B 2024-12-09T20:44:49,762 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:49,762 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b46d5304354f6bfcad03af835fcf16f1, store=C 2024-12-09T20:44:49,762 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T20:44:49,770 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099bee46e3f8184c2cb66fc5166f602138_b46d5304354f6bfcad03af835fcf16f1 is 50, key is test_row_0/A:col10/1733777081621/Put/seqid=0 2024-12-09T20:44:49,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742437_1613 (size=9914) 2024-12-09T20:44:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T20:44:50,176 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:44:50,182 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099bee46e3f8184c2cb66fc5166f602138_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099bee46e3f8184c2cb66fc5166f602138_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:50,183 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/800fc7f370fe4e9db3804f03ad7f2ad2, store: [table=TestAcidGuarantees family=A region=b46d5304354f6bfcad03af835fcf16f1] 2024-12-09T20:44:50,184 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/800fc7f370fe4e9db3804f03ad7f2ad2 is 175, key is test_row_0/A:col10/1733777081621/Put/seqid=0 2024-12-09T20:44:50,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742438_1614 (size=22561) 2024-12-09T20:44:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T20:44:50,589 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/800fc7f370fe4e9db3804f03ad7f2ad2 2024-12-09T20:44:50,599 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/ec4d415a668f44768b18fe80a1dfe757 is 50, key is test_row_0/B:col10/1733777081621/Put/seqid=0 2024-12-09T20:44:50,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742439_1615 (size=9857) 2024-12-09T20:44:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T20:44:51,004 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/ec4d415a668f44768b18fe80a1dfe757 2024-12-09T20:44:51,009 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/1497e1f9f4e44af7a31ec4a8da10d036 is 50, key is test_row_0/C:col10/1733777081621/Put/seqid=0 2024-12-09T20:44:51,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742440_1616 (size=9857) 2024-12-09T20:44:51,012 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/1497e1f9f4e44af7a31ec4a8da10d036 2024-12-09T20:44:51,015 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/A/800fc7f370fe4e9db3804f03ad7f2ad2 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/800fc7f370fe4e9db3804f03ad7f2ad2 2024-12-09T20:44:51,017 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/800fc7f370fe4e9db3804f03ad7f2ad2, entries=100, sequenceid=305, filesize=22.0 K 2024-12-09T20:44:51,017 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/B/ec4d415a668f44768b18fe80a1dfe757 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/ec4d415a668f44768b18fe80a1dfe757 2024-12-09T20:44:51,019 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/ec4d415a668f44768b18fe80a1dfe757, entries=100, sequenceid=305, filesize=9.6 K 2024-12-09T20:44:51,020 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/.tmp/C/1497e1f9f4e44af7a31ec4a8da10d036 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/1497e1f9f4e44af7a31ec4a8da10d036 2024-12-09T20:44:51,022 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/1497e1f9f4e44af7a31ec4a8da10d036, entries=100, sequenceid=305, filesize=9.6 K 2024-12-09T20:44:51,023 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for b46d5304354f6bfcad03af835fcf16f1 in 1262ms, sequenceid=305, compaction requested=false 2024-12-09T20:44:51,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/aaad6036ae3342de957e1b6efea83ef8, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/bcb93efd14fa4b2fa36c5d9b4000419c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/183162e9f6b34511b4dcea217a94253b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dbf0c933a3d24802b1a738d01a56bf57, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2218caab7e454ab1b6fd1486261a8020, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/6534a00b56b6477daa893e403d8a2d51, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/98d0c0c900444a81ac2664ec87ea18ed, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/67d5fcdda8c04c0ca776c55cd6916884, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2ba76249f5684f4ab8c2034c0e77f249, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7a90ff3d74dc42478e58dc66f457b72b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/8b433149c2c648a19ee1ad4488713cab, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/f87b4c6ac4564d1eb0cebbaef2416912, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/75b0d2f1303e433c8ca891b05d3bf08d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/02eaf79767c142fd8f4a0bd98de55f6d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/18a206f1c66148b0bb036ec3195237bc, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/407aa1a62648485ea5249a5512b1e376, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/fa623ae2cca144f5b52cd40e33fa0f47, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7cc710e3df7a4e6a8e3fb1a56aa835b7, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/01a2f47eb18a4ad3a585ade2f9747d6d, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/33910e8357984118bcc6864f6b5a65f3] to archive 2024-12-09T20:44:51,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:44:51,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/aaad6036ae3342de957e1b6efea83ef8 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/aaad6036ae3342de957e1b6efea83ef8 2024-12-09T20:44:51,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/bcb93efd14fa4b2fa36c5d9b4000419c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/bcb93efd14fa4b2fa36c5d9b4000419c 2024-12-09T20:44:51,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/183162e9f6b34511b4dcea217a94253b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/183162e9f6b34511b4dcea217a94253b 2024-12-09T20:44:51,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dbf0c933a3d24802b1a738d01a56bf57 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dbf0c933a3d24802b1a738d01a56bf57 2024-12-09T20:44:51,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2218caab7e454ab1b6fd1486261a8020 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2218caab7e454ab1b6fd1486261a8020 2024-12-09T20:44:51,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/6534a00b56b6477daa893e403d8a2d51 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/6534a00b56b6477daa893e403d8a2d51 2024-12-09T20:44:51,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/98d0c0c900444a81ac2664ec87ea18ed to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/98d0c0c900444a81ac2664ec87ea18ed 2024-12-09T20:44:51,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/67d5fcdda8c04c0ca776c55cd6916884 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/67d5fcdda8c04c0ca776c55cd6916884 2024-12-09T20:44:51,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2ba76249f5684f4ab8c2034c0e77f249 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/2ba76249f5684f4ab8c2034c0e77f249 2024-12-09T20:44:51,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7a90ff3d74dc42478e58dc66f457b72b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7a90ff3d74dc42478e58dc66f457b72b 2024-12-09T20:44:51,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/8b433149c2c648a19ee1ad4488713cab to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/8b433149c2c648a19ee1ad4488713cab 2024-12-09T20:44:51,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/f87b4c6ac4564d1eb0cebbaef2416912 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/f87b4c6ac4564d1eb0cebbaef2416912 2024-12-09T20:44:51,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/75b0d2f1303e433c8ca891b05d3bf08d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/75b0d2f1303e433c8ca891b05d3bf08d 2024-12-09T20:44:51,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/02eaf79767c142fd8f4a0bd98de55f6d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/02eaf79767c142fd8f4a0bd98de55f6d 2024-12-09T20:44:51,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/18a206f1c66148b0bb036ec3195237bc to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/18a206f1c66148b0bb036ec3195237bc 2024-12-09T20:44:51,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/407aa1a62648485ea5249a5512b1e376 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/407aa1a62648485ea5249a5512b1e376 2024-12-09T20:44:51,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/fa623ae2cca144f5b52cd40e33fa0f47 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/fa623ae2cca144f5b52cd40e33fa0f47 2024-12-09T20:44:51,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7cc710e3df7a4e6a8e3fb1a56aa835b7 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/7cc710e3df7a4e6a8e3fb1a56aa835b7 2024-12-09T20:44:51,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/01a2f47eb18a4ad3a585ade2f9747d6d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/01a2f47eb18a4ad3a585ade2f9747d6d 2024-12-09T20:44:51,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/33910e8357984118bcc6864f6b5a65f3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/33910e8357984118bcc6864f6b5a65f3 2024-12-09T20:44:51,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/2f4526c92fd44b04b9ae39a6b5d4ba32, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4911d148d21449b683b11319802cb4b3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/371deb168daf4412bd6d855dfa24a53b, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/cf5344b7ddea47fc9e8e96310091efe6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/3bc537c640714a0c82f93460cfecc695, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/54c4dcc8e1394315bea8fd4813260d8f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/acea48f1ad6340919d583c6cefb59549, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6cd5400576324f11a68af9dc4b5c16a2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5e675a09db604d41a5a6a2943d78abbd, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/9d9244ac5bfb4763a3671e3d883e8651, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c8f3d488cfea4e3c979bbfecaef29617, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/63ba642a7edd42efb32569ca37651648, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/758eb3f6848b46098843457455e634ac, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/70f40bb3d72b472e81a16dc56d90f5ec, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5cf4a088215149f9b67b3f189bf03375, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4dd07499f9ab4d72a7ce54b7e2786c11, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/95c09d7d59f945c089938223988259ea, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c17a49db74344110aae5f53299a95619, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/e1cbd8525b614e7baf835a935809cde2, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6f8fccc5d02743faa66109c04385e8a5] to archive 2024-12-09T20:44:51,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:44:51,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/2f4526c92fd44b04b9ae39a6b5d4ba32 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/2f4526c92fd44b04b9ae39a6b5d4ba32 2024-12-09T20:44:51,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4911d148d21449b683b11319802cb4b3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4911d148d21449b683b11319802cb4b3 2024-12-09T20:44:51,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/371deb168daf4412bd6d855dfa24a53b to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/371deb168daf4412bd6d855dfa24a53b 2024-12-09T20:44:51,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/cf5344b7ddea47fc9e8e96310091efe6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/cf5344b7ddea47fc9e8e96310091efe6 2024-12-09T20:44:51,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/3bc537c640714a0c82f93460cfecc695 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/3bc537c640714a0c82f93460cfecc695 2024-12-09T20:44:51,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/54c4dcc8e1394315bea8fd4813260d8f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/54c4dcc8e1394315bea8fd4813260d8f 2024-12-09T20:44:51,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/acea48f1ad6340919d583c6cefb59549 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/acea48f1ad6340919d583c6cefb59549 2024-12-09T20:44:51,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6cd5400576324f11a68af9dc4b5c16a2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6cd5400576324f11a68af9dc4b5c16a2 2024-12-09T20:44:51,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5e675a09db604d41a5a6a2943d78abbd to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5e675a09db604d41a5a6a2943d78abbd 2024-12-09T20:44:51,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/9d9244ac5bfb4763a3671e3d883e8651 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/9d9244ac5bfb4763a3671e3d883e8651 2024-12-09T20:44:51,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c8f3d488cfea4e3c979bbfecaef29617 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c8f3d488cfea4e3c979bbfecaef29617 2024-12-09T20:44:51,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/63ba642a7edd42efb32569ca37651648 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/63ba642a7edd42efb32569ca37651648 2024-12-09T20:44:51,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/758eb3f6848b46098843457455e634ac to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/758eb3f6848b46098843457455e634ac 2024-12-09T20:44:51,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/70f40bb3d72b472e81a16dc56d90f5ec to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/70f40bb3d72b472e81a16dc56d90f5ec 2024-12-09T20:44:51,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5cf4a088215149f9b67b3f189bf03375 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/5cf4a088215149f9b67b3f189bf03375 2024-12-09T20:44:51,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4dd07499f9ab4d72a7ce54b7e2786c11 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/4dd07499f9ab4d72a7ce54b7e2786c11 2024-12-09T20:44:51,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/95c09d7d59f945c089938223988259ea to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/95c09d7d59f945c089938223988259ea 2024-12-09T20:44:51,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c17a49db74344110aae5f53299a95619 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/c17a49db74344110aae5f53299a95619 2024-12-09T20:44:51,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/e1cbd8525b614e7baf835a935809cde2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/e1cbd8525b614e7baf835a935809cde2 2024-12-09T20:44:51,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6f8fccc5d02743faa66109c04385e8a5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/6f8fccc5d02743faa66109c04385e8a5 2024-12-09T20:44:51,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/8163175e0e5b4ec1a959a81265176ac9, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/d9602334866146d8ad5de2037192fabf, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/78bc81f646da416482c9ae47bd971d22, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/90be9b96b9504481b878710d8613f237, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7968499f29f24874b9f5ed227141ce07, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/0e90a816693c4fba888104e386e460d1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/a4a9c37d9f414d33b7095902d06283ce, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/770d6a6634324abbad18e6d60f622dc1, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c92c8608c37a4f7bab00c2b61e774992, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/310ccc319d5f4311ab832aaee231127e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/9029608684f046139e781d25bcd007d3, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7fb46c04f40b4b70b25d7f4751a68316, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/5ca39a7d4de8425f8e1fb4c32d11e8e6, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/e156cb5395ed400f94f78111a3c85060, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/ea5670bedf60422ca4237aa6a7ee2746, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b3077a6e549546eeba4379acf92bce4c, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7330440b8dee41599eac0479dce6de7e, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b16c9d45f0fa4d06aaf646a367bdf153, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/012a179bf7cd4da286aeb152fcc03b3f, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c313098b01e042e5bbcbe4126434e7e5] to archive 2024-12-09T20:44:51,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T20:44:51,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/8163175e0e5b4ec1a959a81265176ac9 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/8163175e0e5b4ec1a959a81265176ac9 2024-12-09T20:44:51,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/d9602334866146d8ad5de2037192fabf to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/d9602334866146d8ad5de2037192fabf 2024-12-09T20:44:51,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/78bc81f646da416482c9ae47bd971d22 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/78bc81f646da416482c9ae47bd971d22 2024-12-09T20:44:51,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/90be9b96b9504481b878710d8613f237 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/90be9b96b9504481b878710d8613f237 2024-12-09T20:44:51,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7968499f29f24874b9f5ed227141ce07 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7968499f29f24874b9f5ed227141ce07 2024-12-09T20:44:51,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/0e90a816693c4fba888104e386e460d1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/0e90a816693c4fba888104e386e460d1 2024-12-09T20:44:51,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/a4a9c37d9f414d33b7095902d06283ce to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/a4a9c37d9f414d33b7095902d06283ce 2024-12-09T20:44:51,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/770d6a6634324abbad18e6d60f622dc1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/770d6a6634324abbad18e6d60f622dc1 2024-12-09T20:44:51,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c92c8608c37a4f7bab00c2b61e774992 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c92c8608c37a4f7bab00c2b61e774992 2024-12-09T20:44:51,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/310ccc319d5f4311ab832aaee231127e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/310ccc319d5f4311ab832aaee231127e 2024-12-09T20:44:51,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/9029608684f046139e781d25bcd007d3 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/9029608684f046139e781d25bcd007d3 2024-12-09T20:44:51,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7fb46c04f40b4b70b25d7f4751a68316 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7fb46c04f40b4b70b25d7f4751a68316 2024-12-09T20:44:51,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/5ca39a7d4de8425f8e1fb4c32d11e8e6 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/5ca39a7d4de8425f8e1fb4c32d11e8e6 2024-12-09T20:44:51,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/e156cb5395ed400f94f78111a3c85060 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/e156cb5395ed400f94f78111a3c85060 2024-12-09T20:44:51,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/ea5670bedf60422ca4237aa6a7ee2746 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/ea5670bedf60422ca4237aa6a7ee2746 2024-12-09T20:44:51,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b3077a6e549546eeba4379acf92bce4c to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b3077a6e549546eeba4379acf92bce4c 2024-12-09T20:44:51,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7330440b8dee41599eac0479dce6de7e to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/7330440b8dee41599eac0479dce6de7e 2024-12-09T20:44:51,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b16c9d45f0fa4d06aaf646a367bdf153 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/b16c9d45f0fa4d06aaf646a367bdf153 2024-12-09T20:44:51,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/012a179bf7cd4da286aeb152fcc03b3f to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/012a179bf7cd4da286aeb152fcc03b3f 2024-12-09T20:44:51,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c313098b01e042e5bbcbe4126434e7e5 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/c313098b01e042e5bbcbe4126434e7e5 2024-12-09T20:44:51,068 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/recovered.edits/308.seqid, newMaxSeqId=308, maxSeqId=4 2024-12-09T20:44:51,068 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. 2024-12-09T20:44:51,068 DEBUG [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for b46d5304354f6bfcad03af835fcf16f1: Waiting for close lock at 1733777089761Running coprocessor pre-close hooks at 1733777089761Disabling compacts and flushes for region at 1733777089761Disabling writes for close at 1733777089761Obtaining lock to block concurrent updates at 1733777089761Preparing flush snapshotting stores in b46d5304354f6bfcad03af835fcf16f1 at 1733777089761Finished memstore snapshotting TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733777089762 (+1 ms)Flushing stores of TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1. at 1733777089763 (+1 ms)Flushing b46d5304354f6bfcad03af835fcf16f1/A: creating writer at 1733777089763Flushing mob file b46d5304354f6bfcad03af835fcf16f1/A: appending metadata at 1733777089770 (+7 ms)Flushing mob file b46d5304354f6bfcad03af835fcf16f1/A: closing flushed file at 1733777089770Flushing b46d5304354f6bfcad03af835fcf16f1/A: appending metadata at 1733777090183 (+413 ms)Flushing b46d5304354f6bfcad03af835fcf16f1/A: closing flushed file at 1733777090184 (+1 ms)Flushing b46d5304354f6bfcad03af835fcf16f1/B: creating writer at 1733777090595 (+411 ms)Flushing b46d5304354f6bfcad03af835fcf16f1/B: appending metadata at 1733777090599 (+4 ms)Flushing b46d5304354f6bfcad03af835fcf16f1/B: closing flushed file at 1733777090599Flushing b46d5304354f6bfcad03af835fcf16f1/C: creating writer at 1733777091007 (+408 ms)Flushing b46d5304354f6bfcad03af835fcf16f1/C: appending metadata at 1733777091009 (+2 ms)Flushing b46d5304354f6bfcad03af835fcf16f1/C: closing flushed file at 1733777091009Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36379a03: reopening flushed file at 1733777091014 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e7cd8ea: reopening flushed file at 1733777091017 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@82cff6: reopening flushed file at 1733777091020 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for b46d5304354f6bfcad03af835fcf16f1 in 1262ms, sequenceid=305, compaction requested=false at 1733777091023 (+3 ms)Writing region close event to WAL at 1733777091066 (+43 ms)Running coprocessor post-close hooks at 1733777091068 (+2 ms)Closed at 1733777091068 2024-12-09T20:44:51,070 INFO [RS_CLOSE_REGION-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,070 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=b46d5304354f6bfcad03af835fcf16f1, regionState=CLOSED 2024-12-09T20:44:51,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 because future has completed 2024-12-09T20:44:51,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-09T20:44:51,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure b46d5304354f6bfcad03af835fcf16f1, server=00c23a7251f8,36519,1733776895755 in 1.4640 sec 2024-12-09T20:44:51,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-12-09T20:44:51,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b46d5304354f6bfcad03af835fcf16f1, UNASSIGN in 1.4690 sec 2024-12-09T20:44:51,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-09T20:44:51,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4730 sec 2024-12-09T20:44:51,077 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733777091077"}]},"ts":"1733777091077"} 2024-12-09T20:44:51,078 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T20:44:51,078 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T20:44:51,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4830 sec 2024-12-09T20:44:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T20:44:51,752 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:51,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T20:44:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:51,757 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-09T20:44:51,758 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:51,763 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,768 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C, FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/recovered.edits] 2024-12-09T20:44:51,772 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/800fc7f370fe4e9db3804f03ad7f2ad2 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/800fc7f370fe4e9db3804f03ad7f2ad2 2024-12-09T20:44:51,774 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dde494b7708e4b47ae16eaea90c5a64d to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/A/dde494b7708e4b47ae16eaea90c5a64d 2024-12-09T20:44:51,777 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/47620506bc52414690a198799efdcba0 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/47620506bc52414690a198799efdcba0 2024-12-09T20:44:51,778 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/ec4d415a668f44768b18fe80a1dfe757 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/B/ec4d415a668f44768b18fe80a1dfe757 2024-12-09T20:44:51,781 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/0a9f7b995b264091ad1cd14e1571cb42 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/0a9f7b995b264091ad1cd14e1571cb42 2024-12-09T20:44:51,782 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/1497e1f9f4e44af7a31ec4a8da10d036 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/C/1497e1f9f4e44af7a31ec4a8da10d036 2024-12-09T20:44:51,786 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/recovered.edits/308.seqid to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1/recovered.edits/308.seqid 2024-12-09T20:44:51,786 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/default/TestAcidGuarantees/b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,786 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T20:44:51,787 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T20:44:51,788 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T20:44:51,791 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209296d46a29d4a43ddb68d7c94a127609b_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209296d46a29d4a43ddb68d7c94a127609b_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,793 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092b1ce865fb784d4e97339e6a374d26bc_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092b1ce865fb784d4e97339e6a374d26bc_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,794 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120930d99b42b1c4440aa6500910edccee3d_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120930d99b42b1c4440aa6500910edccee3d_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,796 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093dba33f097044bd49ac74e042bb76f1f_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093dba33f097044bd49ac74e042bb76f1f_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,797 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094d20382b80af426eba83346db21d2620_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094d20382b80af426eba83346db21d2620_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,799 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094e832443dd1547f1bfd8839092c0b297_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094e832443dd1547f1bfd8839092c0b297_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,800 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095b58f0f73c504917849bb68ce957bf02_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095b58f0f73c504917849bb68ce957bf02_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,801 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120960a7e792fcfc46589b2badd95e296fe6_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120960a7e792fcfc46589b2badd95e296fe6_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,802 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120981bb73fd5fb74cea9b20f5dddca24f47_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120981bb73fd5fb74cea9b20f5dddca24f47_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,802 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209870b974c0c2d4b0caa16cfd8bdf567d8_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209870b974c0c2d4b0caa16cfd8bdf567d8_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,803 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120987406c7d9cb642269acc69732083d563_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120987406c7d9cb642269acc69732083d563_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,804 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098a45a60b5d4143f3a98f50d0fff2e75e_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098a45a60b5d4143f3a98f50d0fff2e75e_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,804 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098cfbde579ab84c57a2fe5de7d73487ba_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098cfbde579ab84c57a2fe5de7d73487ba_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,805 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099bee46e3f8184c2cb66fc5166f602138_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099bee46e3f8184c2cb66fc5166f602138_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,806 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d01663a71a72481cad5349a3c7785b02_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d01663a71a72481cad5349a3c7785b02_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,806 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e11b0270c47246c9aefd17dd70f35d1e_b46d5304354f6bfcad03af835fcf16f1 to hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e11b0270c47246c9aefd17dd70f35d1e_b46d5304354f6bfcad03af835fcf16f1 2024-12-09T20:44:51,807 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T20:44:51,808 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:51,810 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T20:44:51,812 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T20:44:51,813 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:51,813 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T20:44:51,813 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733777091813"}]},"ts":"9223372036854775807"} 2024-12-09T20:44:51,815 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T20:44:51,815 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b46d5304354f6bfcad03af835fcf16f1, NAME => 'TestAcidGuarantees,,1733777058296.b46d5304354f6bfcad03af835fcf16f1.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T20:44:51,815 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T20:44:51,815 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733777091815"}]},"ts":"9223372036854775807"} 2024-12-09T20:44:51,816 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T20:44:51,816 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T20:44:51,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-12-09T20:44:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-09T20:44:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T20:44:51,864 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T20:44:51,872 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 247), OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=880 (was 536) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4020 (was 4010) - AvailableMemoryMB LEAK? - 2024-12-09T20:44:51,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T20:44:51,872 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:44:51,872 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:44:51,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:51,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:51,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:44:51,873 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T20:44:51,873 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1872569587, stopped=false 2024-12-09T20:44:51,873 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c23a7251f8,42351,1733776895022 2024-12-09T20:44:51,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T20:44:51,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T20:44:51,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:44:51,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:44:51,918 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T20:44:51,919 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:44:51,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T20:44:51,919 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:44:51,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:51,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T20:44:51,920 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c23a7251f8,36519,1733776895755' ***** 2024-12-09T20:44:51,920 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T20:44:51,922 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T20:44:51,923 INFO [RS:0;00c23a7251f8:36519 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T20:44:51,923 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T20:44:51,923 INFO [RS:0;00c23a7251f8:36519 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T20:44:51,923 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(959): stopping server 00c23a7251f8,36519,1733776895755 2024-12-09T20:44:51,923 INFO [RS:0;00c23a7251f8:36519 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T20:44:51,923 INFO [RS:0;00c23a7251f8:36519 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c23a7251f8:36519. 2024-12-09T20:44:51,923 DEBUG [RS:0;00c23a7251f8:36519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:44:51,923 DEBUG [RS:0;00c23a7251f8:36519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:51,924 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T20:44:51,924 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T20:44:51,924 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T20:44:51,924 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T20:44:51,924 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T20:44:51,924 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T20:44:51,925 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T20:44:51,925 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T20:44:51,925 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T20:44:51,925 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T20:44:51,925 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T20:44:51,925 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T20:44:51,925 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-09T20:44:51,944 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/info/e526e506bbe0471faf2faa161737ad4d is 95, key is TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6./info:/1733776932261/DeleteFamily/seqid=0 2024-12-09T20:44:51,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742441_1617 (size=6514) 2024-12-09T20:44:51,963 INFO [regionserver/00c23a7251f8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T20:44:51,963 INFO [regionserver/00c23a7251f8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T20:44:51,970 INFO [regionserver/00c23a7251f8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T20:44:52,125 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T20:44:52,325 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T20:44:52,348 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/info/e526e506bbe0471faf2faa161737ad4d 2024-12-09T20:44:52,371 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/ns/005c7a992b674746a264a73377624372 is 93, key is TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6./ns:/1733776932224/DeleteFamily/seqid=0 2024-12-09T20:44:52,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742442_1618 (size=5906) 2024-12-09T20:44:52,525 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T20:44:52,725 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T20:44:52,775 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/ns/005c7a992b674746a264a73377624372 2024-12-09T20:44:52,798 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/rep_barrier/8310a50291d14f29abf9e6e756d00659 is 102, key is TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6./rep_barrier:/1733776932224/DeleteFamily/seqid=0 2024-12-09T20:44:52,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742443_1619 (size=6025) 2024-12-09T20:44:52,926 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T20:44:53,126 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T20:44:53,202 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/rep_barrier/8310a50291d14f29abf9e6e756d00659 2024-12-09T20:44:53,218 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/table/75103487321447529f709b4abd9c1211 is 96, key is TestAcidGuarantees,,1733776899456.dd7288626d8fbb1461f9ccf6551f1bf6./table:/1733776932224/DeleteFamily/seqid=0 2024-12-09T20:44:53,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742444_1620 (size=5936) 2024-12-09T20:44:53,326 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T20:44:53,526 DEBUG [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T20:44:53,621 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/table/75103487321447529f709b4abd9c1211 2024-12-09T20:44:53,625 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/info/e526e506bbe0471faf2faa161737ad4d as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/info/e526e506bbe0471faf2faa161737ad4d 2024-12-09T20:44:53,627 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/info/e526e506bbe0471faf2faa161737ad4d, entries=12, sequenceid=90, filesize=6.4 K 2024-12-09T20:44:53,627 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/ns/005c7a992b674746a264a73377624372 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/ns/005c7a992b674746a264a73377624372 2024-12-09T20:44:53,630 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/ns/005c7a992b674746a264a73377624372, entries=8, sequenceid=90, filesize=5.8 K 2024-12-09T20:44:53,630 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/rep_barrier/8310a50291d14f29abf9e6e756d00659 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/rep_barrier/8310a50291d14f29abf9e6e756d00659 2024-12-09T20:44:53,633 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/rep_barrier/8310a50291d14f29abf9e6e756d00659, entries=6, sequenceid=90, filesize=5.9 K 2024-12-09T20:44:53,634 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/.tmp/table/75103487321447529f709b4abd9c1211 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/table/75103487321447529f709b4abd9c1211 2024-12-09T20:44:53,636 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/table/75103487321447529f709b4abd9c1211, entries=7, sequenceid=90, filesize=5.8 K 2024-12-09T20:44:53,637 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1712ms, sequenceid=90, compaction requested=false 2024-12-09T20:44:53,641 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-09T20:44:53,641 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T20:44:53,641 INFO [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T20:44:53,641 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733777091924Running coprocessor pre-close hooks at 1733777091925 (+1 ms)Disabling compacts and flushes for region at 1733777091925Disabling writes for close at 1733777091925Obtaining lock to block concurrent updates at 1733777091925Preparing flush snapshotting stores in 1588230740 at 1733777091925Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733777091925Flushing stores of hbase:meta,,1.1588230740 at 1733777091926 (+1 ms)Flushing 1588230740/info: creating writer at 1733777091926Flushing 1588230740/info: appending metadata at 1733777091944 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733777091944Flushing 1588230740/ns: creating writer at 1733777092352 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733777092370 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733777092370Flushing 1588230740/rep_barrier: creating writer at 1733777092779 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733777092798 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733777092798Flushing 1588230740/table: creating writer at 1733777093205 (+407 ms)Flushing 1588230740/table: appending metadata at 1733777093217 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733777093217Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15f29737: reopening flushed file at 1733777093624 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38c89327: reopening flushed file at 1733777093627 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7afe9b64: reopening flushed file at 1733777093630 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@784c2473: reopening flushed file at 1733777093633 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1712ms, sequenceid=90, compaction requested=false at 1733777093637 (+4 ms)Writing region close event to WAL at 1733777093638 (+1 ms)Running coprocessor post-close hooks at 1733777093641 (+3 ms)Closed at 1733777093641 2024-12-09T20:44:53,641 DEBUG [RS_CLOSE_META-regionserver/00c23a7251f8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T20:44:53,727 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(976): stopping server 00c23a7251f8,36519,1733776895755; all regions closed. 2024-12-09T20:44:53,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741834_1010 (size=25344) 2024-12-09T20:44:53,733 DEBUG [RS:0;00c23a7251f8:36519 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/oldWALs 2024-12-09T20:44:53,733 INFO [RS:0;00c23a7251f8:36519 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c23a7251f8%2C36519%2C1733776895755.meta:.meta(num 1733776899069) 2024-12-09T20:44:53,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741832_1008 (size=23221548) 2024-12-09T20:44:53,736 DEBUG [RS:0;00c23a7251f8:36519 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/oldWALs 2024-12-09T20:44:53,736 INFO [RS:0;00c23a7251f8:36519 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c23a7251f8%2C36519%2C1733776895755:(num 1733776898113) 2024-12-09T20:44:53,736 DEBUG [RS:0;00c23a7251f8:36519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:44:53,736 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T20:44:53,736 INFO [RS:0;00c23a7251f8:36519 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T20:44:53,737 INFO [RS:0;00c23a7251f8:36519 {}] hbase.ChoreService(370): Chore service for: regionserver/00c23a7251f8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T20:44:53,737 INFO [RS:0;00c23a7251f8:36519 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T20:44:53,737 INFO [regionserver/00c23a7251f8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T20:44:53,738 INFO [RS:0;00c23a7251f8:36519 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36519 2024-12-09T20:44:53,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c23a7251f8,36519,1733776895755 2024-12-09T20:44:53,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T20:44:53,749 INFO [RS:0;00c23a7251f8:36519 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T20:44:53,749 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c23a7251f8,36519,1733776895755] 2024-12-09T20:44:53,769 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c23a7251f8,36519,1733776895755 already deleted, retry=false 2024-12-09T20:44:53,770 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c23a7251f8,36519,1733776895755 expired; onlineServers=0 2024-12-09T20:44:53,770 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c23a7251f8,42351,1733776895022' ***** 2024-12-09T20:44:53,770 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T20:44:53,770 INFO [M:0;00c23a7251f8:42351 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T20:44:53,770 INFO [M:0;00c23a7251f8:42351 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T20:44:53,770 DEBUG [M:0;00c23a7251f8:42351 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T20:44:53,770 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T20:44:53,770 DEBUG [M:0;00c23a7251f8:42351 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T20:44:53,770 DEBUG [master/00c23a7251f8:0:becomeActiveMaster-HFileCleaner.large.0-1733776897754 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c23a7251f8:0:becomeActiveMaster-HFileCleaner.large.0-1733776897754,5,FailOnTimeoutGroup] 2024-12-09T20:44:53,770 DEBUG [master/00c23a7251f8:0:becomeActiveMaster-HFileCleaner.small.0-1733776897755 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c23a7251f8:0:becomeActiveMaster-HFileCleaner.small.0-1733776897755,5,FailOnTimeoutGroup] 2024-12-09T20:44:53,770 INFO [M:0;00c23a7251f8:42351 {}] hbase.ChoreService(370): Chore service for: master/00c23a7251f8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T20:44:53,770 INFO [M:0;00c23a7251f8:42351 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T20:44:53,771 DEBUG [M:0;00c23a7251f8:42351 {}] master.HMaster(1795): Stopping service threads 2024-12-09T20:44:53,771 INFO [M:0;00c23a7251f8:42351 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T20:44:53,771 INFO [M:0;00c23a7251f8:42351 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T20:44:53,771 ERROR [M:0;00c23a7251f8:42351 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:42427 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:42427,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-09T20:44:53,771 INFO [M:0;00c23a7251f8:42351 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T20:44:53,771 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T20:44:53,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T20:44:53,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:44:53,780 DEBUG [M:0;00c23a7251f8:42351 {}] zookeeper.ZKUtil(347): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T20:44:53,780 WARN [M:0;00c23a7251f8:42351 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T20:44:53,781 INFO [M:0;00c23a7251f8:42351 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/.lastflushedseqids 2024-12-09T20:44:53,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742445_1621 (size=72) 2024-12-09T20:44:53,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T20:44:53,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36519-0x1000c4fefa70001, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T20:44:53,860 INFO [RS:0;00c23a7251f8:36519 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T20:44:53,860 INFO [RS:0;00c23a7251f8:36519 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c23a7251f8,36519,1733776895755; zookeeper connection closed. 2024-12-09T20:44:53,860 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@294d05f6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@294d05f6 2024-12-09T20:44:53,861 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T20:44:54,192 INFO [M:0;00c23a7251f8:42351 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T20:44:54,192 INFO [M:0;00c23a7251f8:42351 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T20:44:54,192 DEBUG [M:0;00c23a7251f8:42351 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T20:44:54,192 INFO [M:0;00c23a7251f8:42351 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:44:54,192 DEBUG [M:0;00c23a7251f8:42351 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:44:54,192 DEBUG [M:0;00c23a7251f8:42351 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T20:44:54,192 DEBUG [M:0;00c23a7251f8:42351 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:44:54,192 INFO [M:0;00c23a7251f8:42351 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=780.28 KB heapSize=956.20 KB 2024-12-09T20:44:54,208 DEBUG [M:0;00c23a7251f8:42351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/13b6baf219ee490487044601c9092187 is 82, key is hbase:meta,,1/info:regioninfo/1733776899148/Put/seqid=0 2024-12-09T20:44:54,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742446_1622 (size=5672) 2024-12-09T20:44:54,213 INFO [M:0;00c23a7251f8:42351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2172 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/13b6baf219ee490487044601c9092187 2024-12-09T20:44:54,233 DEBUG [M:0;00c23a7251f8:42351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c9e25234a02149d4a98d9d06ce02f43c is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x5C/proc:d/1733776996131/Put/seqid=0 2024-12-09T20:44:54,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742447_1623 (size=45764) 2024-12-09T20:44:54,637 INFO [M:0;00c23a7251f8:42351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=779.73 KB at sequenceid=2172 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c9e25234a02149d4a98d9d06ce02f43c 2024-12-09T20:44:54,640 INFO [M:0;00c23a7251f8:42351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c9e25234a02149d4a98d9d06ce02f43c 2024-12-09T20:44:54,659 DEBUG [M:0;00c23a7251f8:42351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b2406c5afd3472db0689553df78a929 is 69, key is 00c23a7251f8,36519,1733776895755/rs:state/1733776897890/Put/seqid=0 2024-12-09T20:44:54,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073742448_1624 (size=5156) 2024-12-09T20:44:55,062 INFO [M:0;00c23a7251f8:42351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2172 (bloomFilter=true), to=hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b2406c5afd3472db0689553df78a929 2024-12-09T20:44:55,065 DEBUG [M:0;00c23a7251f8:42351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/13b6baf219ee490487044601c9092187 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/13b6baf219ee490487044601c9092187 2024-12-09T20:44:55,068 INFO [M:0;00c23a7251f8:42351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/13b6baf219ee490487044601c9092187, entries=8, sequenceid=2172, filesize=5.5 K 2024-12-09T20:44:55,068 DEBUG [M:0;00c23a7251f8:42351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c9e25234a02149d4a98d9d06ce02f43c as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c9e25234a02149d4a98d9d06ce02f43c 2024-12-09T20:44:55,071 INFO [M:0;00c23a7251f8:42351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c9e25234a02149d4a98d9d06ce02f43c 2024-12-09T20:44:55,071 INFO [M:0;00c23a7251f8:42351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c9e25234a02149d4a98d9d06ce02f43c, entries=172, sequenceid=2172, filesize=44.7 K 2024-12-09T20:44:55,072 DEBUG [M:0;00c23a7251f8:42351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b2406c5afd3472db0689553df78a929 as hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8b2406c5afd3472db0689553df78a929 2024-12-09T20:44:55,075 INFO [M:0;00c23a7251f8:42351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42427/user/jenkins/test-data/09f012c2-4264-7277-7ba0-c6ba4d0b4948/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8b2406c5afd3472db0689553df78a929, entries=1, sequenceid=2172, filesize=5.0 K 2024-12-09T20:44:55,076 INFO [M:0;00c23a7251f8:42351 {}] regionserver.HRegion(3140): Finished flush of dataSize ~780.28 KB/799009, heapSize ~955.90 KB/978840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 883ms, sequenceid=2172, compaction requested=false 2024-12-09T20:44:55,077 INFO [M:0;00c23a7251f8:42351 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:44:55,077 DEBUG [M:0;00c23a7251f8:42351 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733777094192Disabling compacts and flushes for region at 1733777094192Disabling writes for close at 1733777094192Obtaining lock to block concurrent updates at 1733777094192Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733777094192Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=799009, getHeapSize=979080, getOffHeapSize=0, getCellsCount=2369 at 1733777094193 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733777094193Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733777094193Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733777094207 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733777094207Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733777094216 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733777094232 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733777094232Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733777094640 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733777094658 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733777094658Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a642860: reopening flushed file at 1733777095065 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@427013c8: reopening flushed file at 1733777095068 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dc7cb0a: reopening flushed file at 1733777095072 (+4 ms)Finished flush of dataSize ~780.28 KB/799009, heapSize ~955.90 KB/978840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 883ms, sequenceid=2172, compaction requested=false at 1733777095076 (+4 ms)Writing region close event to WAL at 1733777095077 (+1 ms)Closed at 1733777095077 2024-12-09T20:44:55,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741830_1006 (size=941396) 2024-12-09T20:44:55,080 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T20:44:55,080 INFO [M:0;00c23a7251f8:42351 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T20:44:55,080 INFO [M:0;00c23a7251f8:42351 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42351 2024-12-09T20:44:55,080 INFO [M:0;00c23a7251f8:42351 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T20:44:55,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T20:44:55,196 INFO [M:0;00c23a7251f8:42351 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T20:44:55,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x1000c4fefa70000, quorum=127.0.0.1:63081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T20:44:55,201 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T20:44:55,204 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T20:44:55,204 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T20:44:55,205 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T20:44:55,205 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/hadoop.log.dir/,STOPPED} 2024-12-09T20:44:55,208 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T20:44:55,208 WARN [BP-1706245004-172.17.0.2-1733776890246 heartbeating to localhost/127.0.0.1:42427 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T20:44:55,208 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T20:44:55,208 WARN [BP-1706245004-172.17.0.2-1733776890246 heartbeating to localhost/127.0.0.1:42427 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1706245004-172.17.0.2-1733776890246 (Datanode Uuid ffb89470-f156-4695-847b-640b1bfe0f55) service to localhost/127.0.0.1:42427 2024-12-09T20:44:55,210 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe/data/data1/current/BP-1706245004-172.17.0.2-1733776890246 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T20:44:55,210 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/cluster_867f1fda-f0d4-c405-5a92-9c8c2f9148fe/data/data2/current/BP-1706245004-172.17.0.2-1733776890246 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T20:44:55,211 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T20:44:55,216 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T20:44:55,217 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T20:44:55,217 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T20:44:55,217 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T20:44:55,217 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/68615c51-421c-fe0f-96f6-6e44b1d05a77/hadoop.log.dir/,STOPPED} 2024-12-09T20:44:55,233 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T20:44:55,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-09T20:44:55,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T20:44:55,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T20:44:55,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T20:44:55,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees